diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f0b6c04 --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +**/.DS_Store +**/_out +go.mod \ No newline at end of file diff --git a/.pipeline/assets/encoding/dvm_cloud_init.t b/.pipeline/assets/encoding/dvm_cloud_init.t deleted file mode 100644 index d235bb5..0000000 --- a/.pipeline/assets/encoding/dvm_cloud_init.t +++ /dev/null @@ -1,9 +0,0 @@ -#cloud-config - -write_files: -- path: "/opt/azure/containers/script.sh" - permissions: "0744" - encoding: gzip - owner: "root" - content: !!binary | - SCRIPT_PLACEHOLDER \ No newline at end of file diff --git a/.pipeline/assets/encoding/encode_cse_script.ps1 b/.pipeline/assets/encoding/encode_cse_script.ps1 deleted file mode 100644 index 2c18d83..0000000 --- a/.pipeline/assets/encoding/encode_cse_script.ps1 +++ /dev/null @@ -1,68 +0,0 @@ -function ConvertTo-GzipData { - [cmdletBinding()] - param( - [parameter(Mandatory = $true, ValueFromPipeline = $true)] - [byte[]]$Data - ) - - $output = [System.IO.MemoryStream]::new() - $gzipStream = New-Object System.IO.Compression.GzipStream $output, ([IO.Compression.CompressionLevel]::Optimal) - - $gzipStream.Write($Data, 0, $Data.Length) - $gzipStream.Close() - return $output.ToArray() -} - -function ConvertTo-Base64String { - [cmdletBinding()] - param( - [parameter(Mandatory = $true, ValueFromPipeline = $true)] - [byte[]]$Data - ) - return [Convert]::ToBase64String($Data) -} - -function ConvertTo-Bytes { - [cmdletBinding()] - param( - [parameter(Mandatory = $true, ValueFromPipeline = $true)] - [string]$Data - ) - $enc = [system.Text.Encoding]::Unicode - return $enc.GetBytes($Data) -} - -function Replace-NewLine { - [cmdletBinding()] - param( - [parameter(Mandatory = $true, ValueFromPipeline = $true)] - [string]$Data - ) - - $Data = $Data.Replace("`r`n", "`n") - return $Data -} - -function Escape-SingleLine { - [cmdletBinding()] - param( - [parameter(Mandatory = $true, ValueFromPipeline = $true)] - [string]$Data - ) - $Data = $Data.Replace("`\", "\\\\") - $Data = $Data.Replace("`r`n", "\n") - $Data = $Data.Replace("`n", "\n") - $Data = $Data.Replace("`"", "\`"") - return $Data -} - -$scriptstring=Get-Content -Raw .\script.sh | Replace-NewLine -$bytes = ConvertTo-Bytes -Data $scriptstring -$gizipData = ConvertTo-GzipData -Data $bytes -$scriptstring = ConvertTo-Base64String -Data $gizipData -$scriptstring - -$customerString=Get-Content -Raw "encode_CustomData.t" -$customerString= $customerString.Replace("SCRIPT_PLACEHOLDER",$scriptstring) | Replace-NewLine | Escape-SingleLine - -"`"customData`": `"[base64(concat('$customerString'))]`"," > script.b64 \ No newline at end of file diff --git a/.pipeline/assets/encoding/encode_cse_script.sh b/.pipeline/assets/encoding/encode_cse_script.sh deleted file mode 100644 index 38b2b7c..0000000 --- a/.pipeline/assets/encoding/encode_cse_script.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -echo "Do not use this script just yet" -exit 1 - -if [ $# -eq 0 ]; then - echo "Pass the file to encode" - exit 1 -fi - -SCRIPT=$1 -OUTPUT=${SCRIPT%.sh}.b64 -TEMPLATE=$(mktemp) - -cat > $TEMPLATE < $OUTPUT - -rm $TEMPLATE \ No newline at end of file diff --git a/AKSEngine-E2E/AksTest-Scripts/aksEngineScale.sh b/AKSEngine-E2E/AksTest-Scripts/aksEngineScale.sh index 15d07b3..bdb177d 100644 --- a/AKSEngine-E2E/AksTest-Scripts/aksEngineScale.sh +++ b/AKSEngine-E2E/AksTest-Scripts/aksEngineScale.sh @@ -1,4 +1,5 @@ -#! /bin/bash +#!/bin/bash -x + set -e log_level() diff --git a/AKSEngine-E2E/AksTest-Scripts/aksEngineUpgrade.sh b/AKSEngine-E2E/AksTest-Scripts/aksEngineUpgrade.sh index 4988c79..6622b4b 100644 --- a/AKSEngine-E2E/AksTest-Scripts/aksEngineUpgrade.sh +++ b/AKSEngine-E2E/AksTest-Scripts/aksEngineUpgrade.sh @@ -1,4 +1,5 @@ -#! /bin/bash +#!/bin/bash -x + set -e log_level() diff --git a/AKSEngine-E2E/AksTest-Scripts/runAksScript.sh b/AKSEngine-E2E/AksTest-Scripts/runAksScript.sh index b509892..c8fdac0 100644 --- a/AKSEngine-E2E/AksTest-Scripts/runAksScript.sh +++ b/AKSEngine-E2E/AksTest-Scripts/runAksScript.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/bin/bash -x function printUsage { diff --git a/AKSEngine-E2E/Template/MainTemplate.json b/AKSEngine-E2E/Template/MainTemplate.json index 5351d5c..aeb3a8d 100644 --- a/AKSEngine-E2E/Template/MainTemplate.json +++ b/AKSEngine-E2E/Template/MainTemplate.json @@ -166,7 +166,12 @@ }, "kubernetesAzureCloudProviderVersion": { "type": "string", - "defaultValue": "1.11", + "metadata": { + "description": "This is the version for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." + } + }, + "kubernetesAzureCloudProviderRelease": { + "type": "string", "metadata": { "description": "This is the version for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." } @@ -183,10 +188,10 @@ "resourceGroupName": "[resourceGroup().name]", "dnsNameForPublicIP": "[toLower(concat('vmd-dns', parameters('masterProfileDnsPrefix')))]", "location": "[resourceGroup().location]", - "imagePublisher": "Canonical", - "imageOffer": "UbuntuServer", - "imageSku": "16.04-LTS", - "imageVersion": "latest", + "imagePublisher": "microsoft-aks", + "imageOffer": "aks", + "imageSku": "aks-engine-ubuntu-1604-202003", + "imageVersion": "2020.03.19", "vmSize": "Standard_D2_v2", "OSDiskName": "osdisk", "nicName": "[concat('vmd-vnic', uniqueString(resourceGroup().id))]", @@ -254,6 +259,9 @@ "kubernetesAzureCloudProviderVersion": { "value": "[parameters('kubernetesAzureCloudProviderVersion')]" }, + "kubernetesAzureCloudProviderRelease": { + "value": "[parameters('kubernetesAzureCloudProviderRelease')]" + }, "identitySystem": { "value": "[parameters('identitySystem')]" }, diff --git a/AKSEngine-E2E/Template/azuredeploy.json b/AKSEngine-E2E/Template/azuredeploy.json index ad35f9f..ba78fd8 100644 --- a/AKSEngine-E2E/Template/azuredeploy.json +++ b/AKSEngine-E2E/Template/azuredeploy.json @@ -164,20 +164,25 @@ }, "kubernetesAzureCloudProviderVersion": { "type": "string", - "defaultValue": "1.11", "metadata": { "description": "This is the version for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." } + }, + "kubernetesAzureCloudProviderRelease": { + "type": "string", + "metadata": { + "description": "This is the release for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." + } } }, "variables": { "resourceGroupName": "[resourceGroup().name]", "dnsNameForPublicIP": "[toLower(concat('vmd-dns', parameters('masterProfileDnsPrefix')))]", "location": "[resourceGroup().location]", - "imagePublisher": "Canonical", - "imageOffer": "UbuntuServer", - "imageSku": "16.04-LTS", - "imageVersion": "latest", + "imagePublisher": "microsoft-aks", + "imageOffer": "aks", + "imageSku": "aks-engine-ubuntu-1604-202003", + "imageVersion": "2020.03.19", "vmSize": "Standard_D2_v2", "OSDiskName": "osdisk", "nicName": "[concat('vmd-vnic', uniqueString(resourceGroup().id))]", @@ -198,7 +203,7 @@ "tenantSubscriptionId": "[subscription().subscriptionId]", "scriptName": "script", "singleQuote": "'", - "scriptParameters": "[concat('IDENTITY_SYSTEM=','\"',parameters('identitySystem'),'\"',' WINDOWS_AGENT_COUNT=','\"',parameters('windowsAgentPoolProfileCount'),'\"' ,' WINDOWS_AGENT_SIZE=','\"',parameters('windowsAgentPoolProfileVMSize'),'\"',' WINDOWS_ADMIN_USERNAME=','\"',parameters('windowsAdminUsername'),'\"',' WINDOWS_ADMIN_PASSWORD=','\"',parameters('windowsAdminPassword'),'\"',' NETWORK_PLUGIN=','\"',parameters('networkPlugin'),'\"',' AVAILABILITY_PROFILE=','\"',parameters('availabilityProfile'),'\"',' RESOURCE_GROUP_NAME=','\"',variables('resourceGroupName'),'\"',' PUBLICIP_DNS=','\"',variables('dnsNameForPublicIP'),'\"' ,' TENANT_ID=','\"',subscription().tenantId,'\"' ,' TENANT_SUBSCRIPTION_ID=','\"',variables('tenantSubscriptionId'),'\"',' ADMIN_USERNAME=','\"',parameters('linuxAdminUsername'),'\"',' MASTER_DNS_PREFIX=','\"',parameters('masterProfileDnsPrefix'),'\"' ,' AGENT_COUNT=','\"',parameters('agentPoolProfileCount'),'\"' ,' AGENT_SIZE=','\"',parameters('agentPoolProfileVMSize'),'\"' ,' MASTER_COUNT=','\"',parameters('masterPoolProfileCount'),'\"',' MASTER_SIZE=','\"',parameters('masterPoolProfileVMSize'),'\"' ,' SPN_CLIENT_ID=','\"',parameters('servicePrincipalClientId'),'\"' ,' SPN_CLIENT_SECRET=','\"',parameters('servicePrincipalClientSecret'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_VERSION=','\"',parameters('kubernetesAzureCloudProviderVersion'),'\"' ,' REGION_NAME=','\"',variables('location'),'\"' ,' SSH_PUBLICKEY=','\"',parameters('sshPublicKey'),'\"' ,' STORAGE_PROFILE=','\"',parameters('storageProfile'),'\"',' AKSENGINE_NODE_COUNT=','\"',parameters('aksEngineNodeCount'),'\"',' AKSENGINE_UPGRADE_VERSION=','\"',parameters('aksEngineUpgradeVersion'),'\"',' AKSENGINE_API_MODEL=','\"',parameters('aksEngineApiModel'),'\"',' AKSENGINE_REPO=','\"',parameters('aksEngineRepository'),'\"',' AKSENGINE_BRANCH=','\"',parameters('aksEngineBranch'),'\"')]" + "scriptParameters": "[concat('IDENTITY_SYSTEM=','\"',parameters('identitySystem'),'\"',' WINDOWS_AGENT_COUNT=','\"',parameters('windowsAgentPoolProfileCount'),'\"' ,' WINDOWS_AGENT_SIZE=','\"',parameters('windowsAgentPoolProfileVMSize'),'\"',' WINDOWS_ADMIN_USERNAME=','\"',parameters('windowsAdminUsername'),'\"',' WINDOWS_ADMIN_PASSWORD=','\"',parameters('windowsAdminPassword'),'\"',' NETWORK_PLUGIN=','\"',parameters('networkPlugin'),'\"',' AVAILABILITY_PROFILE=','\"',parameters('availabilityProfile'),'\"',' RESOURCE_GROUP_NAME=','\"',variables('resourceGroupName'),'\"',' PUBLICIP_DNS=','\"',variables('dnsNameForPublicIP'),'\"' ,' TENANT_ID=','\"',subscription().tenantId,'\"' ,' TENANT_SUBSCRIPTION_ID=','\"',variables('tenantSubscriptionId'),'\"',' ADMIN_USERNAME=','\"',parameters('linuxAdminUsername'),'\"',' MASTER_DNS_PREFIX=','\"',parameters('masterProfileDnsPrefix'),'\"' ,' AGENT_COUNT=','\"',parameters('agentPoolProfileCount'),'\"' ,' AGENT_SIZE=','\"',parameters('agentPoolProfileVMSize'),'\"' ,' MASTER_COUNT=','\"',parameters('masterPoolProfileCount'),'\"',' MASTER_SIZE=','\"',parameters('masterPoolProfileVMSize'),'\"' ,' SPN_CLIENT_ID=','\"',parameters('servicePrincipalClientId'),'\"' ,' SPN_CLIENT_SECRET=','\"',parameters('servicePrincipalClientSecret'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_RELEASE=','\"',parameters('kubernetesAzureCloudProviderRelease'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_VERSION=','\"',parameters('kubernetesAzureCloudProviderVersion'),'\"' ,' REGION_NAME=','\"',variables('location'),'\"' ,' SSH_PUBLICKEY=','\"',parameters('sshPublicKey'),'\"' ,' STORAGE_PROFILE=','\"',parameters('storageProfile'),'\"',' AKSENGINE_NODE_COUNT=','\"',parameters('aksEngineNodeCount'),'\"',' AKSENGINE_UPGRADE_VERSION=','\"',parameters('aksEngineUpgradeVersion'),'\"',' AKSENGINE_API_MODEL=','\"',parameters('aksEngineApiModel'),'\"',' AKSENGINE_REPO=','\"',parameters('aksEngineRepository'),'\"',' AKSENGINE_BRANCH=','\"',parameters('aksEngineBranch'),'\"')]" }, "resources": [ { @@ -311,7 +316,7 @@ "vmSize": "[variables('vmSize')]" }, "osProfile": { - "customData": "[base64(concat('#cloud-config\n\nwrite_files:\n- path: \"/opt/azure/containers/script.sh\"\n permissions: \"0744\"\n encoding: gzip\n owner: \"root\"\n content: !!binary |\n H4sIAAAAAAAA/+x8e3fbNvLo//wUszTPOtkNJTtt93bVKr2KRDs6liWtHknTNkeBSEhCTQEsAfqRx3e/BwBJ8Skp3bp79/x+/iORyMFgMBjMGzr5CzSXhDaXiG/AxveG4Uwmi854tugPp7POYLCY9a+d0XzW/ifs/k5gRraYRQII5QL5PqFrCPFvEQmxBygQECD3Bq0x1+iups6iN3ozHIw6vfb5GRT+TuACET8KMXjsjvoMeRJf52pqO3RNKIYloSgkOWzOeDB62z5/XsSVxebGlO0wnXLwcOCzB2ABDpEgjCqcM+d6POjMslR+9UUUCrwNfCSwwtYfvu4M+r1F59IZzhbd0Xw4W7zuDOZO+/xrje0lExt4Q6jH7jgg6sGA0Oge0BpTAbfIjzAQDh9wyBTCbqfrTNItaT8vcnBH3pbdSsq6HXBxKMiKuAlR186s0+vMOgtn2BuP+sNZ+6uz/WwTGwxbLJCHBAJMvYARKmL56C+uRz1n0P66tJl5XMuI+JpX4z5smYd9iLj8HnEcAqFBFGP8abroDkbz3mLiXPanM2fS/uZsL3XoA7g+izwI8ZpwgcNUcufjntzKVHCrJfcOESHxrFgoJdZeYwFR4CGBQTBw2TbwscCGcXJyYpwAwPc82m5R+PBCfZN/A7bmwCIRRAKWDxCEOMBULVZhkdvqszX4+Bb7IB4C/MQJQxY+gzsUUkLXz4DQFQMWwi0Ol4zjpw09UzM31fcBCtEWKNritnluvpg9BIpEn60b8OQ18omnRYYDCnELbKzWhPVc9p36lk5pE/VdzvwMsM9xOvn3TTVRxaSNRsN8MdILxe6GARchoetGfkSIRRRS/mLIKP6+mXzT7/C9iwN52uK3u+/J2C0Kb/iLLvJ97MEdERtCgaMtBu6GJBC8IVFqILUlPlsvFGefPDU+GnI7XMQxmNa5CYQayX7b+Kkm2bSeyF15Cj+rTXgH0AITrI//t/X8cwr93Xe7gXelgW80D98dGEhKA/t0xd7BoRn/9hSKA1/rrdkzI+bINT7vk9IJFuEDrMktplKqt1Isl8kDGm2XOAS2AkG2mAOhmo13GDboFsOGCED0AVb68B0ln8MUZyinbtTL1XPzxRtEhJoblljcYUwPDlLC2I0XIhgsMeB77EYCe/9RgVR0u1tvQVaLmF2paMp3BPO2da53TeqetvVcf1HHUXLeesLxb3AOVgz+9DvwWLrhcv+/+vwd/AzWD2Dj3+AM4B389a+wDDG6gU+fgPsYB2BJ5BqzxyjWn9LTIs+/mXDPifkGFtH73zC/2ytMStlI3UZWsCK+5DzhQmk6ImCDOFBGldmC5YPAvAH9ldRNchBhFALEOebSSGGuEQgGFN+Bz1wFcZSATVkUuhgu5Pgh2uL9EtbDXBCq568bcpykdBGVCxZKGrR+BZd5GM7Tc6PW5DEsGSFi7rAQOPmQGvTGHyFs7ga7NwtEvYXk5QJ9iELMBXJvFi6jK7KOtG/z5CloAVT/kBX8DDYH6xzewXfSvO/0ZF5ALlhEPTi1zk/lygIkNnBqjd/0TvVWc1iHGAkcgtig3H6bWin5HFdhxmCqHVCIC1wiNJlCHgfNKvAxXYtNA5TKhg0KpIEFL5LWZ+daKC8i42DIbfEj6RKAh1eEEiVZZkqR2sNzffbIjj15FlxrR2qV0isYnFrPTzUaHnkMtreSldbz6hOm3AcPXMmpvAOkcCoZ0kqYC7QNlEUOtyimVZ3C/z2E/18fwpinWJ1Def5YiBex5C12kpdaAXX+/pKcwPoDiMHsluQ3I4nlpREhF4a3gXhoQIfGbNHnRa5lI8duESVB5GtpJILDr5xJl4AKTMXe45EK+l5Ho8uCByXoUhlNpTKCkDGRDUSkpKEgCFkQEvlVsaxa3P5c440pj0Lc3VHK013LH+y5jBDkKjOr4pXLMneapfPTfOJMZ53u1WIyGs0WMpjrX/S7MkqZjuaTrrMYd2av2mbzFoVNnyybd0iFgs0sRY0Ab82D+HrOdJZgi3jYlIfab/INCnHTRXaW7Cb6wDMTNNxQmHXqUG6uWjOsQraFU+vINcV68yiKTyGjXN0Ajp1jP2CK/ov246I/vHQm44mMk9+zAFPOfbj/5uyfYEtP7VjKbMpkrGmvCF3jMAgJFe+P28J/g4B0xfXTV+zwlzGldRQPMgPML5yyyIQDExbBM4Ksg3q7IPy79ydaY0F30AceYFdCgLtBdI15Q/sz9wELBUycf82d6Wy66HYWL+fD3sBpN7Fwm5z7TYmaFw+YPFE7iebYk6uW6v0XC/0uZCYoGExvScjoFlNRLdROzP9yzse0Zs6wM5ylT5pJhqeZZHj4Dygg9i0OOWG0/fzs/Bv77Nw+O9csdaPQz29F7WTKXMBLxIkLHhaI+FxFmhsM/IELvDXy4jCJqIyvIVbSgDi0wHpyt2FoS56aRgF8qnBk/SYJrVgdSccDbPTUNN50+jOVClpMne5o2Ju2n58VEL3JZIKsEjxw7DLqcfVa0y112hpLn1Uw6b0tfanzpQo1jTgIK6GRZuaP/yvyxP6D/swC3s7kGpSvhgUOefHto83au+4PF/OpMxl2rp0W1P9ZedASol0idh8WhWgHWo1l2v9pLykZLBK0hORq6gwv+0Nnl0StwWZVgNZjeznpDLuv6gmziqD1qIajnrOPWVYVaD26iTMe7d+7HGg9ovn4ctLpOYvXzmTaHw1LOK1a0BLO153+oPOyP+jP3i7Gk9FFf1Czp1YVaBFdv+cMZ/L99O105lzvWaxVAC1iuvp2ulC6VSfDx5PR637PmVQs2ToAWsR83ZnOnMkxZ8DKgtag6Q2ni/HEuej/WIvLKoHW4DriQFkZ0CKWoTN7M5pcLcaD+WW/LBRZLHnQIqLx/OWg3+2PJcUHOJQFrUVz8a/eXnIyaCRoEc/EueyPhotDmk/iyYCWscTW+XIymo/3YbMqQIvYptNXC030lfN2L1VV6nw6G006l87+A6dpKYAWMcU+TL93SBGDlYLW4JjOX067k/54JjlYjdCqBi0ifNMf9kZvpgttisad6fTNaFKJsIo5+dH7bJ5VDVqL8KDtsypA92Pbd2StMqhpGI/k/pxAXxegIQixrSrQnEjvviodD9/A+Zn2xQsFP/vBGHe6V51LZ9o2t+gGQ4Du4dfftLcrgUWIKJdRgL0RIuDg86UdYh8jjoGzlbhDIbaDkAXSY8fcdtl2yyh4JNzSdQhrGgVr80ii4po6WB8Toj5LCh+FhcbJFLsqvyRYOnEaDzWME7XcVrPpMZc3tsQNmVxuw2XbJqZ2xJsq96z/tVUKuhlxHGYfJLGE5M+K+Jjb6qXrk+cnLqMUu8IWzM6MOGJa1yfx1DHV8XjXJzYKxA+3BN+1d498GUIJwzjpF6HrN0Vtvs0HkBCDblBjy5sxCsWl7qDfw0v4pHdwifjGQB/ATpZsqJxfWrDJJv0KLoTGqTOkbJUJST1GcWP3NUasIqPsTE9NI027F1LuiPjYq9zfOF2vE34r8mintELGLlnDMAwVx6tsVXPDtrEYSfkxdIb9xiMhWClUc0mo4XqZJxruTh6bVGT8xpqxtY+VtKxZc83OG+dfNb5u+IRG9zbaev/4uiFQ2Fh/0MMFCsHuQmEisO8/rGDfYCM5tumq1q4LSn+o8zrFAqxXo2sHblFIVJiYNsnIaHLN0u08b5w/BxaClNPQiBMOcmiJMcYJSLwyis7kAdIZeDL4clTJ1/StXGvxrWoFWrMESCGwNKKWpYdIkFb8TH9W2/Anik5Tbnam9SbEPGCcCBY+qCJIJmeeduXkD9slFqLQv5NBYkoWz5jHEuuQ6PlliKi7SVIZLdjyXd3tjoU3PkMefwboA98i1ZBSq1nWRIDrM4pToV0TsYmWSmIL8RDYy3LwZshTgG64jRX52dNiB1k55qGbxa1OvpFWr3ZwO1wHR+f2ZB2LIgrIrrhVw24UkEwFzIyPj9aytuIVfJN8sj3sowfJK9veontbtQj84wxsDvYK7BGY1eFxdzS86F8upNOoPNi2aX2sADw5+Vvzs2kYmTyWHjmfdKR71z7AhAy/mjshaPzKGa3FqZrMfidisQ0SdgVgFReZy8fl5jSMpOi0qgeqM0sYzF3RMjlKqvC0UvVhQgHfB9gV2Etrh6U0LwbTKQK16mkpWaRM0ex30N+Lu/WwBxVLIRwQ1WUzJZKNKupnCfTvof6RlCLMtWYqFqFCst6IlFBAXPLlAVC4MzzGiTI90pWQyrJzNdXtbsoNkx+1c6Dqd4bh/DiT0cVARajyLOVC1mYzFwk3soFoQzXnoSBofDaNQs65bSZqb4soWmNpwPKDrdzEpjGdX1z0f3SmiyQwTFHtGbYbdeW8fd2ZD2aSynbjFkX+3vkM+U86xUKjaacLKgA7w9f9yWh4LaMdpXJ2BdCuHGJ05rNXznDW72o1cO3MXo16bdcnmIoFx26IhVFM5OamAMJbRY5UBrMJyWpA4ZlZHRM9P4OvzqBc/3w82b2W/pGcDt7fvJdnkFBlQgIkNsZJlUUsacalz5ZNbWRVhNCU4VZEiXhoepInMgyz1xHxcGPrnWDqyfhC/Ye54IZxNX/pdGeDJFfWPm+cf9P4P4ah2+/a7RfQy7T53kRL7Ao/9dWsj4Xxn+H7dtvMmrNdzMAFC9Eaxy4pCghXS5I4Q4oF5kkY2Uz+vy3jV36ZckCbygFtJiTZDDJVXgkVv4mJ2WyZB3+/rwPamZTjrVJcQW/elJAemDPnVeZfJk7kY4lcxmFR/Ti42I0jVeLOG2xID/CChFyAarQCsioO0K0Xqn9WqvuGYSTlr/b7nEdzfga/v+T23kiV0OWkM361U3vvlahayaQAn+DX3+C0sQ5RsHFitKfwCUQItge/mFlUncHAmbytx4F8H4cPB7B0urP+a2fR60+c7mw0eZsWAvu9WswoknZakLjPCEUewdTF/PTns3fVs4xHk1lnUEIY45PyhPwsofcoXPP3RWUaAwByBbnF4JEQuypeCDHXfUzEkzoTrGOW1zBjt6SYzYd2G8xO72Jq5hvsTmAWki2YyFtxUzdQYOrJMN9na+lFJfSRVdzKI5FUZgnUG+A5B/xJRy1r0UuWtUgW/Aym4+GiO+irTJzTnTizp5DrEExa9urXfVDkCpuqllQlO/BJl8AxnPKmZMXfrGYzB/C+qkeguGU7Zu3fr53dg5qGB91NKCmBJXJvlPNTOxvi8dakETyhmSyNdL7iewhJ4z+hx/PW+ngY6LOktGIdF4QiH4rDdpTr6yTqmoNc6pFsi2mvTCrFktjpHS2IJan788TukYRKeveF478YjN44k5hgs1Tq0zSd/tyKggCHrXen8rPP7tTn94+YIs+F6xSHyv5NrndxkL6RI0MBbf+exXopfp4Kuc6uqOtT0mFLxjcyXtufmyZ+9fblpN9LqkTScH5rn30lbefmYRkSr1gMS05oEuboLkV9jUgvdol9dpfck2jsydiXrh+BTaXk5GMBsO1EtOzE1tg66AlLfgHYNo9WK3Jvx05jOhas2gBoN+gGP6jIxvYozwzIxj5ZcrQttNMzsKOPeMfZwX3YjjlEe6lRjkwNTXlfqFhlxCIWWSUj2STpUfvJsajcSqOiERTHY2KJ1T3JSt+2wMoL51FzJ+Wo5AgUkTyejkjzd2tMS41Wcf5OKw+/orkedpUv6Ry5SOzJ1nyCX4xffwPbRuEaCo5eZpPjB6eNDG434oJtVVQ91hyKXcD5ZADt0uDT3FSZeD9XMofTRpoxaefenOZpLTU0VPQ45OnVIWpCqkf5OMQrci+nKY1MJ9uDwWURFaftXIdGJY3T/k9OrmtiH9bb7ZR8wO0seGHl+Upzsdsqj1uFqQlq5G0JnXMcqoa8dnFkYZpcewGY1sfcg8/mnok43zSCaOkT9wo/8J/P3jVu8EMPCSQnzaEpLq0qOQNW9eNDwpj3Sa6x2DBPrboKW3HxO0+93wMr/zU3McfhLXHxOCTUJQHyU+lQGaW+mjE3vH4mHRPkwONHR82ok1eF+TSCwpxX306TbMbB1qX83Cx0N5iLEAmWSmz22USnTdpWZorC5Pm+n2If0OHpdsmarvJiGxSLOxbejP1oTWi7gO8UfjFe1Os/VQYwjr08cgDPPjX7SSeirdyF7UezIJkL7Do4NnOtlPCXNphn5p4KeGpQS8ZFWdXsBXnKPJyU4RqZ+O6w3ZFQqVgoBdKRGLtSr+Z6P2PYIqBUlNnuzgqwW0R8tCQ+EQ/jxIpXNRHmhqo+yl5/OpuMwIyWERWRGQPkpFOtf8xYcgI5/L0NP380pXo1W2AqEgLGfPMZmIz3CL+RJF++NFtwfnb2TIY+LvZVDOANtRQTunYoWvrYM1uwQj7Hz8D0CBchM1tgZUh7BqayQPJxkXmfQDB9m/gZmNqi5MHkk2dgojJ/dnDld5/fncZ8OHikUjH4Tx2tmjzD7spQWbQ9RvFB+X7EQlL6ExTZc1vVDvZvn9/cTH/UCa7uhKvrkMsfpTtNUKWn0q7BcHqIhKTrr0hC+vwgCWPE+R0LvSIJCYb/wuPwhZt6R2hWKVdIYwFQK+Vy22EG7Fil/DtUbryJdUr3+bfHK13GZw+BQhqfFjOrcfN8qdG3WZ7Ua9tqlvyP0bV7dZHUtnmkPTkwvW1egVn9wIrG2TAf66LPwYpDrWZO4nddOkhvuB2I4iUeOaKtyxbqe3XC00qrFV940hVBavTByIp4Mq4SD/rSl57yv0dav1ymvrjGdLBQFOeWntTEcs+gMnitKRfFpdwCbYmoHF0zOEDTF8/+IQrxAnmP6TLl208PVcsNozuY60SPczFsF1vFkpJ4dz6dja7jH8BKw/Z2PopPgKvbScxcP4lZiVlH5u1ysJ5CH5g6N2gPnvTSRXt3/yJ5V7hF0a65XbGb0ukM5+PFaLhwfuzP2spW7l6mzO0P+7pxL8vwBG4w6iZtfbvsXsrPcT+tIJilEkLKyakzed3vOovrzrBz6ai08K58d0y63EivEScXZhWmSQZNoSCQUlhfNzycZU87fnXVv2ps/CqFzLca1ObdE8YUKhJJv1R9zSIZeeW8XaS1ifKwbOVizy68vs4ON9M2LZo5BiWe5wdZVQ1f6ZTTV5KUuJ2UeIuQoxSxTjRXMUu/MU5U5yp/oK5qN4Qbyu4orHx0gwEJCELMpRNy14TzbxtnX9uD2RRs4BsW+V78G1wY+WLzACmiTcgo+aB/vDDZsv7w6nK0mF71x20zHrzEoFrOBZOG3YtcDLHnCQOGvJfIR9TFIf907GQ5QLHB4LIwxK6AuFwFrq9/WkbXCLFnGsaaAaa3hqGa4peMCS5CFOiviTW2PfXDeZi6BPMYUlrG2ALaKzAbqoVop1f3Gj/960WZjmb185EPtRckOkXIXY9rA7qIngoIIwrOcwdmmItSk+efcxnayTT7p1ex02b00g3X2qJbC6w9Ze0Clp1qlMN230qAVTapVZP4Lg4uqPgWWIUn5QGxXVKg8ecaoDid3bJy38vARUMCVvlZaVSl2ZZEVb7YO1zTVRxbTW1RlbfAKj4qDcmpbjkg96B0D7lCM7fAqnp88DL0oTvPiXFugZV8LN2pzWvZVqnSt/fy7Bffjz3mKmytFYfs+OLLg1hydqkKUw6gdMO23kdpgbXn7RGIipQdAqm6O5yY0ZauySVf6y4KFwxya3cvuPCmhKB4qdc6cHc3c6G4/sbwH6fZtZETmAt7V1uCv76ILedCvpHxaeQLbhgTZzofzNrWD4bhbqT3YE8gvbIFxQteuv/WnkDUDu/uS6+NE3gZ97hQT0e7RN0d0CRtcEyDsjM8cl3MOeBbTIGs4q4fLtTvrHH1q6DPjJPkTWYgW0FmYSv10y9E/WCZ4DoHIwckTRAu84p3hiZq8foAzAczMw6G9Tew1wK+yrsAsT1OTbz6fibt8/8LAAD//2spYv/jWgAA'))]", + "customData": "[base64(concat('#cloud-config\n\nwrite_files:\n- path: \"/opt/azure/containers/script.sh\"\n permissions: \"0744\"\n encoding: gzip\n owner: \"root\"\n content: !!binary |\n H4sIAAAAAAAA/+x9e3fbtpPo//wUU5qnTrqhZDlJtz+lSleRaEfHsqSV5KRpnKtAJCShpgiWAG0rj+9+DwC+RUpKtu7ePXf9RyoSg8FgMBjMC+zRD/U58epzxFZg4ntNs8bjWXs0nfUGk2m7359Ne5fW8Gra+hekf0cwJWtMQw7EYxy5LvGWEOC/QhJgB5DPwUf2DVpiptBdTKxZd/h20B+2u63GCRT+juAMETcMMDj0znMpcgS+9sXEtLwl8TDMiYcCksNmjfrDd63GaRFXFpsdUZZiOmbgYN+lG6A+DhAn1JM4p9blqN+eZql8+k0Ucrz2XcSxxNYbvGn3e91Z+9waTGed4dVgOnvT7l9ZrcYzhe0V5St4SzyH3jFAngN94oX3gJbY43CL3BADYfAJBzRP3rk1sMbtaW84aDWe59ZjhcHBCxS6HJBPYE0d7IJNQ9cBj3KYY1hiT0wZOxJlp92xxskqt06Li5LOeE1vxWQ7bbBxwMmC2PE8L61pu9uetmfWoDsa9gbT1tOT3SvBVxjWmCMHcQTYc3xKPB6JXG92Oexa/dazLfnI45qHxFXsH/WieYZMPIcMB0A8P4ww/jGZdfrDq+5sbJ33JlNr3Hp+spM69Alsl4YOBHhJGMdBshmuRl3B/mQvlG+GO0S4wLOggdgE5hJzCH0HcQycgk3Xvos51o4SKY6W02o1GvvENl69jOBKPJeT2fnofHZhvUtkNyHz9OcK4i6JHVBGF/yYwfnoHG7wJhHsiLyEedbgTW88HFxag2nreZZMhjnPcQ17tySg3hp7vIhkNB6e9fpW6/npLgR+QBfExWnn/vC8N5i1293W86eZjpyCS5dAPPGr3e6Wjxx17p5NWs+fVfbunk3Ku7c7auNOrl61nj8vI9u2aejxZNuxcM7sgPhybbSjoyPtCAB+ZeF6jYLNS/kk/vp0yYCG3A85zDfgB9jHnhRoKSlCGwj6XHyLXeAbHz+ygoAGT+AOBR7xlk+AeAsKNIBbHMwpw49raqR6bqhffRSgNXhojVt6Q3853fjx3Gvw6A1yiaM0DQMU4CaYWIoGVmOZd/IpGdIk8lmM/ASwy3Ay+K91OVDJoLVaTX85VBPF9ooC4wHxlrV8jwDzMPDYywH18K/1+Em14XsbS35Grelz3HeNghv2soNcFztwR/iKeMDQGoNaClYTKBWQXBKXLmeSs48ea581sRw2Yhh0o6ED8bR4T5v4sSJZNx6JVXkM7+UifABogg7G5/9onn5NoF+8SDvebXV8q3j4YU9HstWx5y3oB9g34k+PodjxjVqaHSNihmztq6Yhn8+WmM+iYzxhSoC5OG9bRuOFVBwz5mLst4zTF8CVPmkZTxUqtiILDj/+WPgh2+Q/Um7EdjMeMfwXNMCIsD9+AQ5NKJPDLGgwEzS51L5hSZPj3yzBNG3qLchSbEETJW2xno1mAKbpUTN6MANs0/Uaew4DcyNYkTLiPRi/gYn/ghOAD4LqeYDRDXz5AtcJDFkIMCLBYqLhwwtxiqWiErErDDxoJC/FDslBSP6BkfIy1xovgzoskqYFkT8d6uGUnWqtrXtsh1wZW4fO/1o3/uNaFxMSa8h0tX4lfP+q7dJfY8yDDSzJLfbEmbYWCmsev/DC9RwHQBdqDLHscoPdYVihWwwrwgF5G1goZXqQ5hokOMUabGrVGudUf/kWES7Hhjnmdxh7eztJNdWJJsKpMJVwxN3/VlUl6bbXzowsZhG7yvZnsohid37LhhOa4enXFzt2QkZoXySSqH4lelScDHrMvUQqYxmr6S+ENG3LWDKTuxVxMSyk7Va/RUHdJfO62PB1AZe+Qj6vu4RxlnlvI3uFZQsK7BW5xVHjy7qDb+te6Lpw+vLHRm7Wcvscv80YQwF2sRBRupCeiyTvWMtv3KfpRvyqaUfQvqXEAR4gj9lE2OxiG0amXrRWTDuCs9CzhQwARzfYg0VA19IEbl9MhE1nUwfDHDHsaHkNkHAnz+Yr0ShND5+DnHxNTxVDLBONEy3WKQrbTBkbrTpf+/VIX0TE1mj4j2tqcyGUUaSitlojJop1gy/AMQZjaybwBZYB9sG0QP8/j96/tT40az89/vLoPbY+BEHtp8eGXtDzHk6k20a8DGX5CVAO/HAHxNOT71L+EdOiXVdn+ndpdmkTCjxkAcIOB3xPGJcGKeGwQgw86kmnFOYbjlkNegthQopOQsh9xBhmwl/ETCHgFDx8J7aUhDhI209oGNgYzkT/AVrj3eq+ixknnhq/qstharuDPDFhLlWzMoPVBm0kh5ick0Mxky614g4NgJFPibte+zs0v73C9s0Mec5M8HKGPgl1wpF9M4s3lpzwo8fwORUNKYsmA6MBH6Aggnk1ckZDz4Fjo3EsZuYjvoJjY/S2e6yWmonNhTgOgK9Qbr2VWOVEN4MZgy5XQCIucIl48RBCxShWgYu9JV/VQFrWsEK+8IPACYWTkHr50qHP+PpiWdxQeOfC8yIekZKVbni5hmqjRVuoRJNeqpjGIqGXUzg2To8VGhY6FNa3gpXGaflxJz15B2zBqXwsQuKUMqQsIsbR2peOU7BGEa3iDPnfTfj/9iaMeIrlPhT7jwZ4FkneLJW85KiW+++HeAdWb0AMemdLfjOSuD01wsXE8Nrnmxq0vYgtar+IuUgLao084oeukkbCGfzJqLDPPY49vnN7JIL+dZfV36H+Rgq6UEYToYwgoJRnY4JC0pDvB9QPiHiULCsXt3/WksYeCwPcSSll+wyszKxY6bQydlf7j6uxNZm2Oxez8XA4nXWs8bR31uu0p9ZsMrwad6zZqD193dITW/YOyUBvPUtRzcdrfS++rjWZxthCFghzF7l1tkIBrtvIzJJdR59YZoCaHXC9Sh2KxZVzVibqsXHgnCK9eRDFx5BRrrYPh46xGzBB/03rcdYbnFvj0bg3mLY+Uh97jLlw//zkX2AK6/dQykyP0lAYtMRb4sAPiMc/HraE/wUCkhlXD1+ywt/GlOZBPMh00L9xyCIT9gxYBM8IsjJ9zYLwp+1HSmNBp98D5mNbQIC9Qt4Ss5qyZ+59GnAYW/95ZU2mk1mnPXt1Nej2rVYdc7vOmFsXqFlxg4kdlUo0w46YtVDv1wb6LmQ6SJhsNLpUqK2I/9vpF92YWoP2YJq8qcfJlnqcbGG/IZ+YtzhghHqt05PGc/OkYZ40FEvtMHDzS1E5mHKEXyFGbHAwR8RlMuyzwsA2jOO1lheHceh5QrVGShoQgyYYj+5WFK3JY10rgE8kjqzdJKAlq0NheICJHuva23ZvKtMds4nVGQ66k9bpSQFR1tU3tuCBYZt6DpPNim6h01REjVNhvc1dofOFCtW1KCKyhUYcM3//X5En5t/0pxfwtseXIG01zHHAiq0PNmr3sjeYXU2s8aB9aTWh+s/Ig24hStOsu7BIRCloOZZJ74+dpGSwCNAtJBcTa3DeG1hpPrMCm1ECWo3t1bg96LyuJswoglajGgy71i5mGWWg1ejG1mi4e+1yoNWIrkbn43bXmr2xxpPecLCF06gE3cL5pt3rt1/1+r3puzjzWDXXEtAiul7XGkxF++TdZGpd7pisUQAtYrr4ZTKTulVlRUfj4Zte1xqXTNnYA3oo5rHVt9oT6xDMEWgR82V7MrXGh+wuIwtagaY7mMxGY+us93slLmMLtALXAVvVyIAWsQys6dvh+GI26l+d97bFLYslD1pENLp61e91eiNB8R4OZUEr0Zz9Z3cnORk0ArSIZ2yd94aD2T6dKvBkQLexROf++Xh4NdqFzSgBLWKbTF7PFNEX1rudVJUdFJPpcNw+t3ZvZUVLAbSIKbKOet19Kh6MBLQCx+Tq1aQz7o2mgoPlCI1y0CLCt71Bd/h2MlOH3Kg9mbwdjksRljEn33vXaWqUg1Yi3HuqGiWgu7Ht2rLGNqiuaWQB79+DvlcfQqsFug4fosBLGhLd1TlWebJzo9Z4liJI4iV7Bm7JfrV/LwmNHtLzea1xosfBmAXRHsiOPIJelB72A2zKQj1GhJukHZBgyqem4MsXFUMyymujiih7aYGgjGObtktkNNORZS/Ys4mMLBcqEeDpCTTg55MTgGttjW4wXGs+uodr7c+/4FqTfsq16GTKJJzw4MwV5z6Da81lczNO6l1rjC74HQqw6QfUFx4XZqZN12vqwbXmkGDtLQO41pZe6C/hWtuaXqEO8mGWSDua4ChTSJNUfuK41rQjObdmve5Qm9XWcflWzabrOvbMkNUlc9W/pswV1EOGg+yL2OkzozIrZspG2yWnRzb1PGxzk1Mz0+OAYW2XREPHNQfJKpvI57/dEnzXSl+5wtflmnbUK0KXpbvhOTROlE9qsj7ExKAbVFuzeoRCcqnT73XxHL4oX3yO2EpDn8CMp6ypRF2c5s7u8VJpVaFsusjEDhzq4Vr6GCGWLmx2pMe6liiBQm4EERc7pesbKQ8VmX1ALVAiY+e0pmmaDLjIsGJ9RdeRGAn50VQq5MYhARgJVH1OPM12Mm8U3J1wmxORcWtLSpcultKypPUlbdQaT2vPai7xwnsTrZ2fn9U4CmrLT6o7RwGYHSgMBOb9pwXs6qx6C5UVz2pp2yCVhrnRxLQ5GK+HlxbcooBIfz6pVRZu/5Imy9moNU6BBiDkNNCiyJDousUY7QgEXr7C2fLBZAQWdz4flvI1aRVzLbbKiuwljYEkAkMhahqqiwBpRu/Ub7kM/6Do1MViZyqgA8x8yginwSbS70lyIymOzm+286iQshyJLlg8pQ6Ns9uxSp8HyLNXccypCWuWJkjvaHDjUuSwJ4A+sTWSRbyVmmVJONgu9XAitEvCV+FcSmzBcQVzvu1la2IXoBtmYkl+dreYflaOWWBnccudryVpxhQuxbW3d25NlpEoIp+kWcgKdqfl4QJIj7aP0rKm5BU8j3+ZDnbRRvDKNNfo3pSFVT+fgMlkGccQ9PI4Rmc4OOudz4QNLh2Clm58LgE8Ovqp/lXXtEzAUfW8iorc9zAhw696KgS1Pxn1KnHKYvrvRMzXfswuH4ziJHOB09yYyo6FHwTTKoGqjiUMeppdjreSzBAuZCKfeIDvfWxz7CRJ3q14PAbdKgI1q2nZOpEy2c3voL8b1ZZjB0qmQhggT+U3pUjWyqifxtDfQ/0DKUW4UpqpmC0MyHLFE0IBMcGXDaAgPXi0I3n0CFNCKMv2xURdEZBmmCxMl8aBTLRqmvX7VDhrfenwi72UiwDU67nAQi3r19dkZT3y/dpXXSskB1p6rPbWyENLLA6wfGcjN7CuTa7Oznq/W5NZ7GcnqHZ0S3tdWO/etK/6U0Flq3aLQnfneJr4JxliptC0kgkVgDP3FJTKSTPVHdFFa19NX1uDaa+j1MClNX097LZsl2CPzxi2A8y1oveSGwIIaxY5UhobiEmWHQrvdK30PDo9ER7PdqL64WT3UthHYjj4ePNR7EHiySPER3ylHZWdiFuace7SeV0dstJDqAu/KvQI39QdwRPhb5nLkDi4tnaOsOcI/0L+BzPONO3i6pXVmfYTj1g6xP+uaarKrdV6Cd3MbaubcI5t7ia2mvG50P8r/Npq6dnjLPUZGKcBWuLIJEU+YXJKAmfgYY5Z7DHW4//ebuOXdpk0QOvSAK3HJJkUMul4ARW1RMSs1tSBf7uvAkqPlMNPpajUoX6zhXTPmDmrMt8YG5EPJXIZg0UWTuFi2ZRQiak1WBMW4BkJGAdZEQdkUeygamTkfRSh7muaFucpWx9zFk3jBL4/N/pRS5TQ+bg9ep2qvY9SVI14UIAv8OdfcFxbBshfWRHaY/gCPADTgWs9i6rd71vjd9U4kOviYLMHS7sz7b2xZt3e2OpMh+N3Sca2163EjEJxTnMSFYSh0CHYszE7fn/yoXyU0XA8bfe3EEb4hDwhN0voPQqW7GNRmUYAgGxObjE4JMC29BcCzFTBGXGEzgTjkOnVovDgVtpFhvPa3bOJnq+EPIJpQNagI2fBdFXpgj1HuPkuXQorKqaPLKKaK4GkNEqg7o2xnAH+qC2nNevG05rFE34Ck9Fg1un3ZGDT6oyt6WPIlXLGtZXV894rcoVFlVMqkx34omoVMByzumDFT0a9ngP4WFbMUVyylFm71ys996CiMkWVfQpKYI7sG2n8VI6GWLQ0iQdPvEyURhhf0d3N+CId8Q7nrfF5P9BXQWnJPM6Ih1wodkspV7d65dVQMdUD2RYX+JcFlSJJbHcPFsQtqfvnxO6BhEpY94XtP+sP31rjiGB9KyeraDp+3wx9HwfND8fit0vv5O+PDxdBgZy7Hl3mbY8vUz9I3WIWroA6/55Eeil6nwh5FD136UYabHH/WsZq+2fDxK/fvRr30ku+pyeNf5knT8XZudrMA+IUc4vxDo3dHFVOqu4Aq8nOsUvv4ttlNb06hLN1ZRtMT0hO3hcA04xFy4zPGlM5PcGWXQCmycLFgtybkdGY9AWj0gFKO93gjfRsTMdjmQ5Z3ydLjjoLzWQPpPQR57BzcBe2QzbRTmqkIVNBU94WKiZtMY9EVspINkh60HoyzEuXsixjhfP3x1XxuNS3TTDywnnQ2FHA0Yy3QBHJw+mIJH63xN5WRVwUv1PKwy25BQFpiksYR/LyUmW05ovKpZkmCpZQMPQyixy9OK5lcNsh43QtveqR4lBkAl6N+9Da6nycGyrj7+cqEOC4lkRMWrmW4zytW/UhJSUjeXqVixqT6nhsFOAFuRfDbPVMBtuBQd77P27lCl5KaZz0/rByRSi7sN6uJ+QTbmXBCzPPJ+6LZXF53NJNjVEjZ028K4YDWTnZKvYsDJOr1gDd+Jx78VXfMRBjq5ofzl1iX+ANe3/yoXaDN13EkRg0h6Y4tbLgDBjlr/cJY94mucR8RR056zJsxcmnlnqvC0b+MTcww8EtsfEoIJ5NfOQm0iEjSj05Yq579UjKJ8iBR68OGlEFrwrjKQSFMS9+mcRVD3srwfJj08BeYcYDxGkisdl3YxU2aRmZIUoGj0Ipewvcvm3wN5HLbmSGKAyer+Eq1nTtHy6NFHWkCV3zML+jwc3IDZfEaxXwHcO19rJa+cochHboFaM9eHbp+HwxQ5R1ebDjK/MRI+WZ67mCW/ihBfqJviP9npzmWyebPNKzH0nyqIPjHGD2FvT+Q09AJWIhtVdbYOzIj7lkCY5gi4BCS2drgEvAbhFx0Zy4hG9GsQlRVmqa6yqrbbu9yXQ8BB3dMDOchx4PzcbPtZNnegSak1PJiRGlsSJg8G8teP9ZF1peb4IuifEpdfUnoFPWJexGEH/+Sm9C4+TkifDAbOyqr0INlDwTb2l5aO5iR2/CArkMPwHdIYwHVG+CkSHyCejyIBSvi2z8ApyqT0E8AV0dbHkw8eYJ6GibUyncdtvXD8cRH/ZurkQg/rs2WUW4I71iti3kDvXwXkl/wHxW8kGy7A4uK/L7L+/k3Eh/114ur2+sqnvMb6U7RVCpwdSqwHC8j4S4lrNIQvJ+LwkjxNgdDZwiCTGG/4Hb4RsX9Y54WfVcIo0FQKWet4tJM2CHqufvULnRIlYp3dNfDle6lE03vkQa7RY9q3HzfKnQt1meVGvbcpb8f6Nrd+oioW3zSLuiY/J1ghLM8rtpCmdNf6iLYXsTH5WaOQ4jqAxGciNyTzBB4JGf0lPZE/lcHnc1kqTJN+50SZDsvdfBI45w7/hGXRJUQ/7PkdZvl6lvTnXtzVdFIa5HFS7lEyj1oSuyVlFGuUBbLCoHpy720PTNo38KAzxDzkOaTPkq2H1Je03r9K9UvMk6G7SKFWtxZr5zNZkOL6MvZybRg1Y+mBADl1e16LmyFr0UswoQtLZjBgn0nqFznXbgSa7StNJbNXFb4W5Mq+LOTDqk1R5cjWbDwcz6vTdtybMybUyY2xv0VP1gluExXH/YiasL0yBjws9RL0lk6KcnjV9ymYyEkxNr/KbXsWaX7UH73JLR6TSLeEjUXkuunccXrCWmcQZNIS+RUFidvtwf7E8Kj1XxQVnfqCmBzFc8VIb/Y8YUEiNx2VZ16iTueWG9myUpku1u2QTKjlV4c5ntrifVYl5mG2zxPN/JKKs7S4acvJYf21VVrcSZBQwliFW8u4xZqkU7kgW0bOPZsuoRbjx658HCRTcYEAc/wEwYIXd1aPxSO3lm9qcTMIGt5Hec1QcUMXL5agMJolVAPfJJfRE4XrLe4OJ8OJtc9EYtPeo8xyAr3zkVB7sT2hgiyxP6FDmvkIs8Gwfsy6GD5QD5CoNNgwDbHKKsGdiu+hSRSlViR9e0JQXs3WrqQs+cUs54gHz1GJ/GZvZiUAQpTsboBDQXoNdkJVOqV3cefuprV5nCavkx8U3lPY12ETItta1BB3nHHILQA+vUgilmfKvW9J+5PG9l7hwkV/eTmvitG9GVub8mGDuy6wUsqWoU3dKnLcCyM6lZEX8vdi6o+CYYhTfbHaJzSYJGvyuAoqh608g9bwMXDxIwtt9t9So9tgVRpQ07uyu6in3LqS2q8iYYxVdbXXKqW3TIvdi6XV6imZtglL3ee3l+3x35+HBughH/3Lopndeyza2E484r0d986/mQC86Vpzhk+xcb92LJnUtlmHIAW/emq22UJhg7Wg9AVKRsH0jZjfD4GG2q1GD8WHX9u3AgN9Pb3oWWLQTFq9rGnhvZmWvi1ffAH+5jDn/fmaGOT44ZN9P8Ffz4MjqTZ6JFeL6hy5mmja3JVX/aMn7TNHsl7BJzDMmdNCjeYFMFxuYYwlZwd7/VrB3Bq6iIx3OUH03k5QhFkvzfVQga5AnGQtvGjAG+xR6QRVTWxLj84h+TX9R9oh3FLZmOdAGZiS3kR4iI/HQeZyq6IzrEVR42dYqXosZy8mprXfWneuRmqycwlxye5o2L6KRPjAf5fCJO/v8bAAD//5kAvuhKZQAA'))]", "computerName": "[variables('vmName')]", "adminUsername": "[parameters('linuxAdminUsername')]", "linuxConfiguration": { diff --git a/AKSEngine-E2E/Template/azuredeploy.parameters.json b/AKSEngine-E2E/Template/azuredeploy.parameters.json index c5d577e..70c8a2b 100644 --- a/AKSEngine-E2E/Template/azuredeploy.parameters.json +++ b/AKSEngine-E2E/Template/azuredeploy.parameters.json @@ -35,6 +35,9 @@ "kubernetesAzureCloudProviderVersion": { "value": "" }, + "kubernetesAzureCloudProviderRelease": { + "value": "" + }, "identitySystem": { "value": "" }, diff --git a/AKSEngine-E2E/Template/azurestack_template.json b/AKSEngine-E2E/Template/azurestack_template.json index df9fec5..be88cc7 100644 --- a/AKSEngine-E2E/Template/azurestack_template.json +++ b/AKSEngine-E2E/Template/azurestack_template.json @@ -4,20 +4,39 @@ "properties": { "orchestratorProfile": { "orchestratorType": "Kubernetes", - "orchestratorRelease": "1.11", + "orchestratorRelease": "", + "orchestratorVersion": "", "kubernetesConfig": { + "cloudProviderBackoff": true, + "cloudProviderBackoffRetries": 1, + "cloudProviderBackoffDuration": 30, + "cloudProviderRateLimit": true, + "cloudProviderRateLimitQPS": 3, + "cloudProviderRateLimitBucket": 10, + "cloudProviderRateLimitQPSWrite": 3, + "cloudProviderRateLimitBucketWrite": 10, "kubernetesImageBase": "mcr.microsoft.com/k8s/azurestack/core/", "useInstanceMetadata": false, - "networkPlugin": "flannel", + "networkPlugin": "kubenet", + "kubeletConfig": { + "--node-status-update-frequency": "1m" + }, + "controllerManagerConfig": { + "--node-monitor-grace-period": "5m", + "--pod-eviction-timeout": "5m", + "--route-reconciliation-period": "1m" + }, "customWindowsPackageURL": "" } }, "customCloudProfile": { - "portalURL": "" + "identitySystem": "", + "portalURL": "", + "authenticationMethod": "" }, "masterProfile": { "dnsPrefix": "", - "distro": "ubuntu", + "distro": "aks-ubuntu-16.04", "osDiskSizeGB": 100, "availabilityProfile": "AvailabilitySet", "count": 3, diff --git a/AKSEngine-E2E/Template/script.sh b/AKSEngine-E2E/Template/script.sh index b1fd6ab..55fb5e4 100644 --- a/AKSEngine-E2E/Template/script.sh +++ b/AKSEngine-E2E/Template/script.sh @@ -1,15 +1,23 @@ -#! /bin/bash -ex +#!/bin/bash -ex ERR_APT_INSTALL_TIMEOUT=9 # Timeout installing required apt packages ERR_AKSE_DOWNLOAD=10 # Failure downloading AKS-Engine binaries ERR_AKSE_DEPLOY=12 # Failure calling AKS-Engine's deploy operation ERR_TEMPLATE_DOWNLOAD=13 # Failure downloading AKS-Engine template ERR_INVALID_AGENT_COUNT_VALUE=14 # Both Windows and Linux agent value is zero +ERR_TEMPLATE_GENERATION=15 # The default api model could not be generated ERR_CACERT_INSTALL=20 # Failure moving CA certificate ERR_METADATA_ENDPOINT=30 # Failure calling the metadata endpoint ERR_API_MODEL=40 # Failure building API model using user input ERR_AZS_CLOUD_REGISTER=50 # Failure calling az cloud register ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete +#ERR_AKSE_GENERATE=11 # Failure calling AKS-Engine's generate operation +#ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT=26 # Timeout waiting for Microsoft's GPG key download +#ERR_AZS_CLOUD_ENVIRONMENT=51 # Failure setting az cloud environment +#ERR_AZS_CLOUD_PROFILE=52 # Failure setting az cloud profile +#ERR_AZS_LOGIN_AAD=53 # Failure to log in to AAD environment +#ERR_AZS_LOGIN_ADFS=54 # Failure to log in to ADFS environment +#ERR_AZS_ACCOUNT_SUB=55 # Failure setting account default subscription ### # @@ -35,6 +43,29 @@ log_level() esac } +apt_get_install() +{ + retries=$1; wait_sleep=$2; timeout=$3; + shift && shift && shift + + for i in $(seq 1 $retries); do + wait_for_apt_locks + dpkg --configure -a + apt-get install --no-install-recommends -y ${@} + [ $? -eq 0 ] && break || \ + if [ $i -eq $retries ]; then + return 1 + else + sleep $wait_sleep + apt_get_update + fi + done + + echo "Executed apt-get install --no-install-recommends -y \"$@\" $i times"; + wait_for_apt_locks +} + + ### # # Retry given command by given number of times in case we have hit any failure. @@ -56,6 +87,42 @@ retrycmd_if_failure() log_level -i "Command Executed $i times."; } +wait_for_apt_locks() +{ + while fuser /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1; do + echo 'Waiting for release of apt locks' + sleep 3 + done +} + +# Avoid transcient apt-update failures +# Function taken from the AKSe's code based +apt_get_update() +{ + log_level -i "Updating apt cache." + + retries=10 + apt_update_output=/tmp/apt-get-update.out + + for i in $(seq 1 $retries); do + wait_for_apt_locks + dpkg --configure -a + apt-get -f -y install + apt-get update 2>&1 | tee $apt_update_output | grep -E "^([WE]:.*)|([eE]rr.*)$" + [ $? -ne 0 ] && cat $apt_update_output && break || \ + cat $apt_update_output + if [ $i -eq $retries ]; then + return 1 + else + sleep 30 + fi + done + + echo "Executed apt-get update $i time/s" + wait_for_apt_locks +} + + ### # # Validate if file exist and it has non zero bytes. If validation passes moves file to new location. @@ -156,6 +223,7 @@ log_level -i "AKSENGINE_UPGRADE_VERSION: $AKSENGINE_UPGRADE_VERSI log_level -i "AVAILABILITY_PROFILE: $AVAILABILITY_PROFILE" log_level -i "IDENTITY_SYSTEM: $IDENTITY_SYSTEM" log_level -i "K8S_AZURE_CLOUDPROVIDER_VERSION: $K8S_AZURE_CLOUDPROVIDER_VERSION" +log_level -i "K8S_AZURE_CLOUDPROVIDER_RELEASE: $K8S_AZURE_CLOUDPROVIDER_RELEASE" log_level -i "MASTER_COUNT: $MASTER_COUNT" log_level -i "MASTER_DNS_PREFIX: $MASTER_DNS_PREFIX" log_level -i "MASTER_SIZE: $MASTER_SIZE" @@ -173,13 +241,32 @@ log_level -i "WINDOWS_ADMIN_USERNAME: $WINDOWS_ADMIN_USERNAME" log_level -i "WINDOWS_AGENT_COUNT: $WINDOWS_AGENT_COUNT" log_level -i "WINDOWS_AGENT_SIZE: $WINDOWS_AGENT_SIZE" +if [[ "$K8S_AZURE_CLOUDPROVIDER_VERSION" == "" ]]; then + if [[ "$K8S_AZURE_CLOUDPROVIDER_RELEASE" == "1.14" ]]; then + K8S_AZURE_CLOUDPROVIDER_VERSION="1.14.7" + else + K8S_AZURE_CLOUDPROVIDER_VERSION="1.15.10" + fi +fi ##################################################################################### # Install pre-requisites -retrycmd_if_failure 5 10 sudo apt-get update -y -PACKAGES="make pax jq curl apt-transport-https lsb-release software-properties-common dirmngr gnupg" -retrycmd_if_failure 5 10 sudo apt-get install ${PACKAGES} -y +log_level -i "Updating apt cache." +apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + +log_level -i "Installing azure-cli and dependencies." +apt_get_install 30 1 600 \ +make \ +pax \ +jq \ +curl \ +apt-transport-https \ +lsb-release \ +software-properties-common \ +dirmngr \ +gnupg \ +|| exit $ERR_APT_INSTALL_TIMEOUT #################################################################################### #Section to install Azure CLI. @@ -317,7 +404,7 @@ IDENTITY_SYSTEM_LOWER=`echo "$IDENTITY_SYSTEM" | tr '[:upper:]' '[:lower:]'` ##################################################################################### # Section to generate ARM template using AKS Engine, login using Azure CLI and deploy the template. # https://docs.microsoft.com/en-us/azure/azure-stack/user/azure-stack-version-profiles-azurecli2#connect-to-azure-stack -HYBRID_PROFILE=2018-03-01-hybrid +HYBRID_PROFILE=2019-03-01-hybrid log_level -i "Register to AzureStack cloud using below command." retrycmd_if_failure 5 10 az cloud register -n $ENVIRONMENT_NAME --endpoint-resource-manager $TENANT_ENDPOINT --suffix-storage-endpoint $SUFFIXES_STORAGE_ENDPOINT --suffix-keyvault-dns $SUFFIXES_KEYVAULT_DNS --endpoint-active-directory-resource-id $ENDPOINT_ACTIVE_DIRECTORY_RESOURCEID --endpoint-active-directory $ENDPOINT_ACTIVE_DIRECTORY_ENDPOINT --endpoint-active-directory-graph-resource-id $ENDPOINT_GRAPH_ENDPOINT log_level -i "Set Azure stack environment." @@ -341,7 +428,8 @@ jq --arg SSH_PUBLICKEY "${SSH_PUBLICKEY}" '.properties.linuxProfile.ssh.publicKe jq --arg AUTHENTICATION_METHOD $AUTHENTICATION_METHOD '.properties.customCloudProfile.authenticationMethod = $AUTHENTICATION_METHOD' | \ jq --arg SPN_CLIENT_ID $SPN_CLIENT_ID '.properties.servicePrincipalProfile.clientId = $SPN_CLIENT_ID' | \ jq --arg SPN_CLIENT_SECRET $SPN_CLIENT_SECRET '.properties.servicePrincipalProfile.secret = $SPN_CLIENT_SECRET' | \ -jq --arg K8S_VERSION $K8S_AZURE_CLOUDPROVIDER_VERSION '.properties.orchestratorProfile.orchestratorRelease=$K8S_VERSION' | \ +jq --arg K8S_RELEASE $K8S_AZURE_CLOUDPROVIDER_RELEASE '.properties.orchestratorProfile.orchestratorRelease=$K8S_RELEASE' | \ +jq --arg K8S_VERSION $K8S_AZURE_CLOUDPROVIDER_VERSION '.properties.orchestratorProfile.orchestratorVersion=$K8S_VERSION' | \ jq --arg NETWORK_PLUGIN $NETWORK_PLUGIN '.properties.orchestratorProfile.kubernetesConfig.networkPlugin=$NETWORK_PLUGIN' \ > $AZURESTACK_CONFIGURATION_TEMP @@ -356,7 +444,7 @@ if [ "$AGENT_COUNT" != "0" ]; then jq --arg linuxAgentCount $AGENT_COUNT \ --arg linuxAgentSize $AGENT_SIZE \ --arg linuxAvailabilityProfile $AVAILABILITY_PROFILE \ - --arg NODE_DISTRO "ubuntu" \ + --arg NODE_DISTRO "aks-ubuntu-16.04" \ '.properties.agentPoolProfiles += [{"name": "linuxpool", "osDiskSizeGB": 100, "AcceleratedNetworkingEnabled": false, "distro": $NODE_DISTRO, "count": $linuxAgentCount | tonumber, "vmSize": $linuxAgentSize, "availabilityProfile": $linuxAvailabilityProfile}]' \ > $AZURESTACK_CONFIGURATION_TEMP @@ -482,6 +570,7 @@ log_level -i "SSH_KEY_NAME: $SSH_KEY_NAME" log_level -i "STORAGE_ENDPOINT_SUFFIX: $STORAGE_ENDPOINT_SUFFIX" log_level -i "SUBSCRIPTION_ID: $SUBSCRIPTION_ID" log_level -i "TENANT_ID: $TENANT_ID" +log_level -i "K8S_AZURE_CLOUDPROVIDER_VERSION: $K8S_AZURE_CLOUDPROVIDER_VERSION" log_level -i "------------------------------------------------------------------------" make test-kubernetes &> deploy_test_results diff --git a/README.md b/README.md index 7abf242..3c31036 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,3 @@ # azurestack-gallery -To hold any artifact related to Gallery items for all resource providers. \ No newline at end of file +To hold any artifact related to Gallery items for all resource providers. diff --git a/diagnosis/Makefile b/diagnosis/Makefile new file mode 100644 index 0000000..0804ca0 --- /dev/null +++ b/diagnosis/Makefile @@ -0,0 +1,6 @@ +VERSION ?= dev + +# Usage: VERSION=1.2.3 make release +.PHONY: release +release: + ./scripts/release.sh ${VERSION} diff --git a/diagnosis/README.md b/diagnosis/README.md index 467b17a..f194c64 100644 --- a/diagnosis/README.md +++ b/diagnosis/README.md @@ -1,24 +1,39 @@ -# Troubleshooting +# Troubleshooting AKS Engine on Azure Stack -This short [guide](https://github.com/Azure/aks-engine/blob/master/docs/howto/troubleshooting.md) from the Azure's AKS Engine team has a good high level explanation of how AKS Engine interacts with the Azure Resource Manager (ARM) and lists a few potential issues that can cause AKS Engine commands to fail. +This short [guide](https://github.com/Azure/aks-engine/blob/master/docs/howto/troubleshooting.md) from Azure's AKS Engine team has a good high level explanation of how AKS Engine interacts with the Azure Resource Manager (ARM) and lists common reasons that can cause AKS Engine commands to fail. That guide applies to Azure Stack as well as it ships with its own ARM instance. If you are facing a problem that is not part of this guide, then you will need extra information to figure out the root cause. -Please refer to this [article](https://docs.microsoft.com/en-us/azure/azure-stack/user/azure-stack-solution-template-kubernetes-trouble) for specifics about how the `Kubernetes Cluster` marketplace item works on Azure Stack. +Typically, to collect logs from servers you manage, you have to start a remote session using SSH and browse for relevant log files. The scripts in this directory are aim to simplify the collection of relevant logs from your Kubernetes cluster. Just download/unzip the latest [release](https://github.com/msazurestackworkloads/azurestack-gallery/releases/tag/diagnosis-v0.1.2) and execute script `getkuberneteslogs.sh`. -## Gathering logs +> Before you execute `getkuberneteslogs.sh`, make sure that you can login to your Azure Stack instance using `Azure CLI`. Follow this [article](https://docs.microsoft.com/azure-stack/user/azure-stack-version-profiles-azurecli2) to learn how to configure Azure CLI to manage your Azure Stack cloud. -The Bash scripts on this directory are aim to simplify the collection of relevant logs from your Kubernetes cluster. Instead of SSH-ing into the cluster nodes, you can simply download and execute script `getkuberneteslogs.sh` and wait for the logs to be saved back into your workstation. +The logs retrieved by `getkuberneteslogs.sh` are the following: -These are the logs retrieved by the script: +- Log files in directory `/var/log/azure/` +- Log file `/var/log/waagent.log` (waagent) +- Log file `/var/log/azure/deploy-script-dvm.log` (if deployed using Azure Stack's Kubernetes Cluster marketplace item) +- Static manifests in directory `/etc/kubernetes/manifests` +- Static addons in directory `/etc/kubernetes/addons` +- kube-system containers metadata and logs +- kubelet status and journal +- etcd status and journal +- docker status and journal +- kube-system snapshot -- Microsoft Azure Linux Agent (waagent) logs -- Cloud-init logs -- Custom Script Extension logs -- Running Docker container metadata -- Running Docker container logs -- Kubelet service status and journal -- Etcd service status and journal -- Gallery item's DVM logs +## Required Parameters -Please be aware that the log collector script needs to update file `~/.ssh/config` in order to connect to the cluster's worker nodes. While the script will try to back it up and then restore it once the process is complete, it may be a good a idea to create your own copy. +`-u, --user` - The administrator username for the cluster VMs -After the log collection process is complete, the script will also try to look for common issues or misconfigurations. If any of those are found, they will be saved in file `ALL_ERRORS.txt`. \ No newline at end of file +`-i, --identity-file` - RSA private key tied to the public key used to create the Kubernetes cluster (usually named 'id_rsa') + +`-g, --resource-group` - Kubernetes cluster resource group + +## Optional Parameters + +`--disable-host-key-checking` - Sets SSH's `StrictHostKeyChecking` option to `no` while the script executes. Only use in a safe environment. + +`--upload-logs` - Persists retrieved logs in an Azure Stack storage account. Logs can be found in `KubernetesLogs` resource group. + +`--api-model` - Persists apimodel.json file in an Azure Stack Storage account. + Upload apimodel.json file to storage account happens when `--upload-logs` parameter is also provided. + +`-h, --help` - Print script usage diff --git a/diagnosis/collectlogs.sh b/diagnosis/collectlogs.sh index 98905d0..808be1e 100755 --- a/diagnosis/collectlogs.sh +++ b/diagnosis/collectlogs.sh @@ -1,112 +1,198 @@ -#! /bin/bash - -NOW=`date +%Y%m%d%H%M%S` -LOGDIRECTORY="$HOSTNAME-$NOW" -LOGFILENAME="kube_logs.tar.gz" -TRACEFILENAME="$LOGDIRECTORY/collector_trace" -ERRFILENAME="$LOGDIRECTORY/ERRORS.txt" -CURRENTUSER=`whoami` - -mkdir -p $LOGDIRECTORY - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Cleaning up old temp logs" | tee -a $TRACEFILENAME -sudo rm -f $LOGFILENAME - -# Loading common functions -source ./common.sh $ERRFILENAME -source ./detectors.sh $ERRFILENAME - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Starting log collection" | tee -a $TRACEFILENAME - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for syslog file" | tee -a $TRACEFILENAME -try_copy_file /var/log/syslog $LOGDIRECTORY/ - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Dumping Microsoft Azure Linux Agent (waagent) directory tree" | tee -a $TRACEFILENAME -try_print_directory_tree /var/lib/waagent $LOGDIRECTORY/waagent.tree +#!/bin/bash + +collectKubeletMetadata() +{ + KUBELET_REPOSITORY=$(docker images --format '{{.Repository}}' | grep hyperkube) + KUBELET_TAG=$(docker images --format '{{.Repository}}:{{.Tag}}' | grep hyperkube | cut -d ":" -f 2) + KUBELET_VERBOSITY=$(cat /etc/systemd/system/kubelet.service | grep -e '--v=[0-9]' -oh | grep -e [0-9] -oh | head -n 1) + KUBELET_LOG_FILE=${LOGDIRECTORY}/daemons/k8s-kubelet.log + + echo "== BEGIN HEADER ==" > ${KUBELET_LOG_FILE} + echo "Type: Daemon" >> ${KUBELET_LOG_FILE} + echo "TenantId: ${TENANT_ID}" >> ${KUBELET_LOG_FILE} + echo "Name: kubelet" >> ${KUBELET_LOG_FILE} + echo "Version: ${KUBELET_TAG}" >> ${KUBELET_LOG_FILE} + echo "Verbosity: ${KUBELET_VERBOSITY}" >> ${KUBELET_LOG_FILE} + echo "Image: ${KUBELET_REPOSITORY}" >> ${KUBELET_LOG_FILE} + echo "SubscriptionID: ${SUB_ID}" >> ${KUBELET_LOG_FILE} + echo "ResourceGroup: ${RESOURCE_GROUP}" >> ${KUBELET_LOG_FILE} + echo "== END HEADER ==" >> ${KUBELET_LOG_FILE} +} + +collectMobyMetadata() +{ + DOCKER_VERSION=$(docker version | grep -A 20 "Server:" | grep "Version:" | head -n 1 | cut -d ":" -f 2 | xargs) + DOCKER_LOG_FILE=${LOGDIRECTORY}/daemons/k8s-docker.log + + echo "== BEGIN HEADER ==" > ${DOCKER_LOG_FILE} + echo "Type: Daemon" >> ${DOCKER_LOG_FILE} + echo "TenantId: ${TENANT_ID}" >> ${DOCKER_LOG_FILE} + echo "Name: docker" >> ${DOCKER_LOG_FILE} + echo "Version: ${DOCKER_VERSION}" >> ${DOCKER_LOG_FILE} + echo "SubscriptionID: ${SUB_ID}" >> ${DOCKER_LOG_FILE} + echo "ResourceGroup: ${RESOURCE_GROUP}" >> ${DOCKER_LOG_FILE} + echo "== END HEADER ==" >> ${DOCKER_LOG_FILE} +} + +collectEtcdMetadata() +{ + ETCD_VERSION=$(/usr/bin/etcd --version | grep "etcd Version:" | cut -d ":" -f 2 | xargs) + ETCD_LOG_FILE=${LOGDIRECTORY}/daemons/k8s-etcd.log + + echo "== BEGIN HEADER ==" > ${ETCD_LOG_FILE} + echo "Type: Daemon" >> ${ETCD_LOG_FILE} + echo "TenantId: ${TENANT_ID}" >> ${ETCD_LOG_FILE} + echo "Name: etcd" >> ${ETCD_LOG_FILE} + echo "Version: ${ETCD_VERSION}" >> ${ETCD_LOG_FILE} + echo "SubscriptionID: ${SUB_ID}" >> ${ETCD_LOG_FILE} + echo "ResourceGroup: ${RESOURCE_GROUP}" >> ${ETCD_LOG_FILE} + echo "== END HEADER ==" >> ${ETCD_LOG_FILE} +} + +collectContainerMetadata() +{ + local cid=$1 + local pname=$2 + local cname=$3 + + CVERBOSITY=$(docker inspect ${cid} | grep -e "--v=[0-9]" -oh | grep -e [0-9] -oh | head -n 1) + IMAGE_SHA=$(docker inspect ${cid} | grep Image | grep -e "sha256:[[:alnum:]]*" -oh | head -n 1 | cut -d ':' -f 2) + IMAGE=$(docker image inspect ${IMAGE_SHA} | jq -r '.[] | .RepoTags | @tsv' | xargs) + CLOG_FILE=${LOGDIRECTORY}/containers/k8s-${pname}-${cname}.log + + echo "== BEGIN HEADER ==" > ${CLOG_FILE} + echo "Type: Container" >> ${CLOG_FILE} + echo "TenantId: ${TENANT_ID}" >> ${CLOG_FILE} + echo "Name: ${cname}" >> ${CLOG_FILE} + echo "Hostname: ${HOSTNAME}" >> ${CLOG_FILE} + echo "ContainerID: ${cid}" >> ${CLOG_FILE} + echo "Image: ${IMAGE}" >> ${CLOG_FILE} + echo "Verbosity: ${CVERBOSITY}" >> ${CLOG_FILE} + echo "SubscriptionID: ${SUB_ID}" >> ${CLOG_FILE} + echo "ResourceGroup: ${RESOURCE_GROUP}" >> ${CLOG_FILE} + echo "== END HEADER ==" >> ${CLOG_FILE} +} + +compressLogsDirectory() +{ + sync + + echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Compressing logs and cleaning up temp files" + CURRENTUSER=$(whoami) + LOGFILENAME="${HOSTNAME}.zip" + sudo rm -f ${LOGFILENAME} + + sudo chown -R ${CURRENTUSER} ${LOGDIRECTORY} + # TODO This wont work on a disconnected scenario + (cd $TMP && zip -q -r ~/${LOGFILENAME} ${HOSTNAME}) + sudo chown ${CURRENTUSER} ~/${LOGFILENAME} +} + +collectCloudProviderJson() { + if [ -f /etc/kubernetes/azure.json ]; then + sudo jq . /etc/kubernetes/azure.json | sudo grep -v aadClient > ${LOGDIRECTORY}/etc/kubernetes/azure.json + fi + if [ -f /etc/kubernetes/azurestackcloud.json ]; then + sudo jq . /etc/kubernetes/azurestackcloud.json > ${LOGDIRECTORY}/etc/kubernetes/azurestackcloud.json + fi +} + +checkNetworking() { + local DIR=${LOGDIRECTORY}/network + mkdir -p ${DIR} + ping ${HOSTNAME} -c 3 &> ${DIR}/ping.txt +} + +TMP=$(mktemp -d) +LOGDIRECTORY=${TMP}/${HOSTNAME} +mkdir -p ${LOGDIRECTORY} + +echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Collecting azure logs" +mkdir -p ${LOGDIRECTORY}/var/log/azure +cd /var/log/azure +for f in *.log +do + sudo cp "$f" ${LOGDIRECTORY}/var/log/azure/k8s-"${f%}" || : +done -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for Microsoft Azure Linux Agent (waagent) log file" | tee -a $TRACEFILENAME -try_copy_file /var/log/waagent.log $LOGDIRECTORY/ +cd /var/log +for f in cloud-init*.log +do + sudo cp "$f" ${LOGDIRECTORY}/var/log/k8s-"${f%}" || : +done -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for cloud-init log files" | tee -a $TRACEFILENAME -try_copy_file /var/log/cloud-init.log $LOGDIRECTORY/ -try_copy_file /var/log/cloud-init-output.log $LOGDIRECTORY/ +sudo cp /var/log/waagent.log ${LOGDIRECTORY}/var/log/k8s-waagent.log || : -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for CSE directory" | tee -a $TRACEFILENAME -mkdir -p $LOGDIRECTORY/cse/ -try_copy_directory_content /var/log/azure/ $LOGDIRECTORY/cse -try_copy_file /opt/m $LOGDIRECTORY/cse/ +if [ -f /var/log/azure/deploy-script-dvm.log ] +then + sudo apt install zip -y + compressLogsDirectory + exit +fi -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for apt logs" | tee -a $TRACEFILENAME -mkdir -p $LOGDIRECTORY/apt/ -try_copy_directory_content /var/log/apt/ $LOGDIRECTORY/apt +echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Collecting static pod manifests" +mkdir -p ${LOGDIRECTORY}/etc/kubernetes/manifests +sudo cp /etc/kubernetes/manifests/* ${LOGDIRECTORY}/etc/kubernetes/manifests 2>/dev/null +mkdir -p ${LOGDIRECTORY}/etc/kubernetes/addons +sudo cp /etc/kubernetes/addons/* ${LOGDIRECTORY}/etc/kubernetes/addons 2>/dev/null -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Dumping running container list" | tee -a $TRACEFILENAME -sudo docker ps &> $LOGDIRECTORY/containers.list +test $# -gt 0 && NAMESPACES=$@ +test -z "${NAMESPACES}" && echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Collecting logs from pods in all namespaces" +test -n "${NAMESPACES}" && echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Collecting logs from pods in these namespaces: $NAMESPACES" +mkdir -p ${LOGDIRECTORY}/containers -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for containers logs" | tee -a $TRACEFILENAME -mkdir -p $LOGDIRECTORY/containers/manifests +TENANT_ID=$(sudo jq -r '.tenantId' /etc/kubernetes/azure.json) +SUB_ID=$(sudo jq -r '.subscriptionId' /etc/kubernetes/azure.json) +RESOURCE_GROUP=$(sudo jq -r '.resourceGroup' /etc/kubernetes/azure.json) -test $# -gt 0 && NAMESPACES=$@ -test -z "${NAMESPACES}" && echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Collection logs from all namespaces" -test -n "${NAMESPACES}" && echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Collection logs from containers in these namespaces: $NAMESPACES." +if [ "${TENANT_ID}" == "adfs" ] +then + TENANT_ID=$(sudo jq -r '.serviceManagementEndpoint' /etc/kubernetes/azurestackcloud.json | cut -d / -f4) +fi for cid in $(docker ps -a -q --no-trunc) do - cns=`docker inspect --format='{{ index .Config.Labels "io.kubernetes.pod.namespace" }}' $cid` + cns=$(docker inspect --format='{{ index .Config.Labels "io.kubernetes.pod.namespace" }}' ${cid}) - # Only collect logs from requested namespaces # if NAMESPACES not set, then collect everything if [ -z "${NAMESPACES}" ] || (echo $NAMESPACES | grep -qw $cns); then - # Ignore the pod's Pause container - if docker inspect --format='{{ .Config.Image }}' $cid | grep -q -v pause-amd64; + # Ignore the Pause container + if docker inspect --format='{{ .Config.Image }}' ${cid} | grep -q -v pause-amd64; then - # TODO Check size - cname=`docker inspect --format='{{ index .Config.Labels "io.kubernetes.pod.name" }}' $cid` - clog=`docker inspect --format='{{ .LogPath }}' $cid` - sudo docker inspect $cid &> $LOGDIRECTORY/containers/$cname.json - sudo cp $clog $LOGDIRECTORY/containers/$cname.log + pname=$(docker inspect --format='{{ index .Config.Labels "io.kubernetes.pod.name" }}' ${cid}) + cname=$(docker inspect --format='{{ index .Config.Labels "io.kubernetes.container.name" }}' ${cid}) + clog=$(docker inspect --format='{{ .LogPath }}' ${cid}) + + collectContainerMetadata ${cid} ${pname} ${cname} + sudo docker inspect ${cid} &> ${LOGDIRECTORY}/containers/k8s-${pname}-${cname}.json + sudo cat $clog >> ${LOGDIRECTORY}/containers/k8s-${pname}-${cname}.log fi fi done -if is_master_node; -then - echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for static pod manifests" | tee -a $TRACEFILENAME - try_copy_directory_content /etc/kubernetes/manifests/ $LOGDIRECTORY/containers/manifests -fi +test -n "${NAMESPACES}" && echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Collecting daemon logs" +mkdir -p ${LOGDIRECTORY}/daemons -if is_master_node; then SERVICES="docker kubelet etcd"; else SERVICES="docker kubelet"; fi +# TODO use --until --since --lines to limit size +if systemctl list-units | grep -q kubelet.service; then + collectKubeletMetadata + sudo journalctl -n 10000 --utc -o short-iso -u kubelet &>> ${LOGDIRECTORY}/daemons/k8s-kubelet.log +fi -for service in $SERVICES -do - echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Dumping $service service status and journal" | tee -a $TRACEFILENAME - if systemctl list-units | grep -q $service.service; then - sudo systemctl show $service &> $LOGDIRECTORY/${service}_status.log - # journal can be long, do not collect everything - # TODO make this smarter - sudo journalctl -u $service | head -n 10000 &> $LOGDIRECTORY/${service}_journal_head.log - sudo journalctl -u $service | tail -n 10000 &> $LOGDIRECTORY/${service}_journal_tail.log - - if systemctl is-active --quiet $service.service | grep inactive; then - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] The $service service is not running" | tee -a $ERRFILENAME - fi - else - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] The $service service is not installed" | tee -a $ERRFILENAME - fi -done +if systemctl list-units | grep -q etcd.service; then + collectEtcdMetadata + sudo journalctl -n 10000 --utc -o short-iso -u etcd &>> ${LOGDIRECTORY}/daemons/k8s-etcd.log +fi -sync +if systemctl list-units | grep -q docker.service; then + collectMobyMetadata + sudo journalctl -n 10000 --utc -o short-iso -u docker &>> ${LOGDIRECTORY}/daemons/k8s-docker.log +fi -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for known issues and misconfigurations" | tee -a $TRACEFILENAME -find_cse_errors $LOGDIRECTORY/cse/cluster-provision.log -find_cse_errors $LOGDIRECTORY/cloud-init-output.log -find_etcd_bad_cert_errors $LOGDIRECTORY/cse/cluster-provision.log $LOGDIRECTORY/etcd_status.log +collectCloudProviderJson -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Compressing logs" | tee -a $TRACEFILENAME -sudo chown -R $CURRENTUSER $LOGDIRECTORY -sudo tar -czf $LOGFILENAME $LOGDIRECTORY -sudo chown $CURRENTUSER $LOGFILENAME +test -n "${NAMESPACES}" && echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Basic networking test" +checkNetworking -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Cleaning up temp files" -sudo rm -rf $LOGDIRECTORY +compressLogsDirectory diff --git a/diagnosis/collectlogsdvm.sh b/diagnosis/collectlogsdvm.sh deleted file mode 100755 index b5aea9f..0000000 --- a/diagnosis/collectlogsdvm.sh +++ /dev/null @@ -1,58 +0,0 @@ -#! /bin/bash - -NOW=`date +%Y%m%d%H%M%S` -LOGDIRECTORY="$HOSTNAME-$NOW" -LOGFILENAME="dvm_logs.tar.gz" -TRACEFILENAME="$LOGDIRECTORY/collector_trace" -ERRFILENAME="$LOGDIRECTORY/ERRORS.txt" -CURRENTUSER=`whoami` - -mkdir $LOGDIRECTORY - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Cleaning up old temp files" | tee -a $TRACEFILENAME -sudo rm -f $LOGFILENAME - -# Loading common functions -source ./common.sh $ERRFILENAME -source ./detectors.sh $ERRFILENAME - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Starting DVM log collection" | tee -a $TRACEFILENAME - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for syslog file" | tee -a $TRACEFILENAME -try_copy_file /var/log/syslog $LOGDIRECTORY/ - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Dumping Microsoft Azure Linux Agent (waagent) directory tree" | tee -a $TRACEFILENAME -try_print_directory_tree /var/lib/waagent $LOGDIRECTORY/waagent.tree - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for Microsoft Azure Linux Agent (waagent) log file" | tee -a $TRACEFILENAME -try_copy_file /var/log/waagent.log $LOGDIRECTORY/ - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for cloud-init log files" | tee -a $TRACEFILENAME -try_copy_file /var/log/cloud-init.log $LOGDIRECTORY/ -try_copy_file /var/log/cloud-init-output.log $LOGDIRECTORY/ - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for CSE directory" | tee -a $TRACEFILENAME -mkdir -p $LOGDIRECTORY/cse/ -try_copy_directory_content /var/log/azure/ $LOGDIRECTORY/cse - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for apt logs" | tee -a $TRACEFILENAME -mkdir -p $LOGDIRECTORY/apt/ -try_copy_directory_content /var/log/apt/ $LOGDIRECTORY/apt - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Dumping system journal" | tee -a $TRACEFILENAME -sudo journalctl &> $LOGDIRECTORY/journalctl.log - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Looking for known issues and misconfigurations" | tee -a $TRACEFILENAME -find_cse_errors $LOGDIRECTORY/cse/cluster-provision.log -find_cse_errors $LOGDIRECTORY/cloud-init-output.log -find_spn_errors $LOGDIRECTORY/deploy-script-dvm.log -# This line goes away after gallery item v0.4.1 is available -find_spn_errors $LOGDIRECTORY/acsengine-kubernetes-dvm.log - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Compressing logs into $LOGFILENAME" | tee -a $TRACEFILENAME -sudo chown -R $CURRENTUSER $LOGDIRECTORY -sudo tar -czf $LOGFILENAME $LOGDIRECTORY -sudo chown $CURRENTUSER $LOGFILENAME - -echo "[$(date +%Y%m%d%H%M%S)][INFO][$HOSTNAME] Cleaning up temp files" -sudo rm -r -f $LOGDIRECTORY \ No newline at end of file diff --git a/diagnosis/common.sh b/diagnosis/common.sh deleted file mode 100644 index 1a2812e..0000000 --- a/diagnosis/common.sh +++ /dev/null @@ -1,79 +0,0 @@ -#! /bin/bash - -ERRFILENAME=$1 - -is_master_node() -{ - if [[ $HOSTNAME == k8s-master* ]]; then - return 0 - else - return 1 - fi -} - -### -# -# Wrapper around the cp command that prints a nice message if the source file does not exist. -# -# Source file expected location -# Destination directory -### -try_copy_file() -{ - if [ -f $1 ]; then - sudo cp $1 $2/ - else - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] Expected file not found: $1" | tee -a $ERRFILENAME - fi -} - -### -# -# Wrapper around the cp command that prints a nice message if the source directory does not exist. -# -# Source directory expected location -# Destination directory -### -try_copy_directory() -{ - if [ -d $1 ]; then - sudo cp -r $1 $2/ - else - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] Expected directory not found: $1" | tee -a $ERRFILENAME - fi -} - -### -# -# Wrapper around the cp command that prints a nice message if the source directory does not exist. -# -# Source directory expected location -# Destination directory -### -try_copy_directory_content() -{ - if [ -d $1 ]; then - for f in $1/* - do - sudo cp -r $f $2/ - done - else - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] Expected directory not found: $1" | tee -a $ERRFILENAME - fi -} - -### -# -# Wrapper around the find command that prints a nice message if the source directory does not exist. -# -# Source directory expected location -# Destination file -### -try_print_directory_tree() -{ - if [ -d $1 ]; then - sudo find $1 &> $2 - else - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] Expected directory not found: $1" | tee -a $ERRFILENAME - fi -} diff --git a/diagnosis/detectors.sh b/diagnosis/detectors.sh deleted file mode 100644 index 037f6a4..0000000 --- a/diagnosis/detectors.sh +++ /dev/null @@ -1,93 +0,0 @@ -#! /bin/bash - -ERRFILENAME=$1 - -### -# -# Look for known errors on CustomScriptExtension logs -# -# CustomScriptExtension logs location -### -find_cse_errors() -{ - if [ -f $1 ]; - then - ERROR=`grep "VMExtensionProvisioningError" $1 -A 1 | tail -n 1` - - if [ "$ERROR" ]; then - echo "====================" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME][VMExtensionProvisioningError] $ERROR" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Hint: The list of error codes can be found here: https://github.com/Azure/aks-engine/blob/master/parts/k8s/kubernetesprovisionsource.sh" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Log file source: $1" | tee -a $ERRFILENAME - fi - fi -} - -### -# -# Look for known errors on DVM logs -# -# DVM logs location -### -find_spn_errors() -{ - if [ -f $1 ]; - then - ERROR403=$(grep "failed to load apimodel" $1 | grep "StatusCode=403") - - if [ "$ERROR403" ]; then - echo "====================" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME][AuthorizationFailed] $ERROR403" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Hint: Double-check the entered Service Principal has write permissions to the target subscription" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Help: https://aka.ms/AzsK8sSpn" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Log file source: $1" | tee -a $ERRFILENAME - fi - - ERROR401=$(grep "failed to load apimodel" $1 | grep "StatusCode=401" | grep "invalid_client") - - if [ "$ERROR401" ]; then - echo "====================" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME][InvalidClient] $ERROR401" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Hint: double-check the entered Service Principal secret is correct" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Log file source: $1" | tee -a $ERRFILENAME - fi - - ERROR400=$(grep "failed to load apimodel" $1 | grep "StatusCode=400" | grep "unauthorized_client") - - if [ "$ERROR400" ]; then - echo "====================" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME][InvalidClient] $ERROR400" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Hint: double-check the entered Service Principal name is correct" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][ERROR][$HOSTNAME] Log file source: $1" | tee -a $ERRFILENAME - fi - fi -} - -### -# -# Look for known errors with etcd certs -# -# CustomScriptExtension logs location -# etcd status log -### -find_etcd_bad_cert_errors() -{ - if [ -f $1 -a -f $2 ] - then - STATUS14=`grep "command terminated with exit status=14" $1` - BAD_CERT=`grep "remote error: tls: bad certificate" $2` - - if [ "$STATUS14" -a "$BAD_CERT" ] - then - TRUSTED_HOSTS=`openssl x509 -in /etc/kubernetes/certs/etcdpeer*.crt -text -noout | grep "X509v3 Subject Alternative Name" -A 1 | tail -n 1 | xargs` - - echo "====================" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME][TlsBadEtcdCertificate] $ERROR" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] Hint: The etcd instance running on $HOSTNAME cannot establish a secure connection with its peers." | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] These are the trusted hosts as listed in certificate /etc/kubernetes/certs/etcdpeer[0-9].crt: "$TRUSTED_HOSTS"." | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] Make sure the etcd peers are running on trusted hosts." | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] Log file source 1: $1" | tee -a $ERRFILENAME - echo "[$(date +%Y%m%d%H%M%S)][WARN][$HOSTNAME] Log file source 2: $2" | tee -a $ERRFILENAME - fi - fi -} diff --git a/diagnosis/dig.sh b/diagnosis/dig.sh deleted file mode 100644 index 155329f..0000000 --- a/diagnosis/dig.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -function download_scripts -{ - ARTIFACTSURL=$1 - - echo "[$(date +%Y%m%d%H%M%S)][INFO] Pulling dependencies from this repo: $ARTIFACTSURL" - - mkdir -p scripts - - for script in detectors - do - if [ -f scripts/$script.sh ]; then - echo "[$(date +%Y%m%d%H%M%S)][INFO] Dependency '$script.sh' already in local file system" - fi - - SCRIPTURL=$ARTIFACTSURL/diagnosis/$script.sh - curl -fs $SCRIPTURL -o scripts/$script.sh - - if [ ! -f scripts/$script.sh ]; then - echo "[$(date +%Y%m%d%H%M%S)][ERROR] Required script not available. URL: $SCRIPTURL" - echo "[$(date +%Y%m%d%H%M%S)][ERROR] You may be running an older version. Download the latest script from github: https://aka.ms/AzsK8sLogCollectorScript" - exit 1 - fi - done -} - -find_dvm_issues() -{ - for host in $(ls $LOGDIRECTORY | grep vmd-* | xargs); - do - echo "[$(date +%Y%m%d%H%M%S)][INFO][$host] Looking for known issues and misconfigurations" - find_cse_errors $LOGDIRECTORY/$host/cse/cluster-provision.log - find_cse_errors $LOGDIRECTORY/$host/cloud-init-output.log - find_spn_errors $LOGDIRECTORY/$host/deploy-script-dvm.log - find_spn_errors $LOGDIRECTORY/$host/acsengine-kubernetes-dvm.log - done -} - -find_node_issues() -{ - for host in $(ls $LOGDIRECTORY | grep k8s-* | xargs); - do - echo "[$(date +%Y%m%d%H%M%S)][INFO][$host] Looking for known issues and misconfigurations" - find_cse_errors $LOGDIRECTORY/$host/cse/cluster-provision.log - find_cse_errors $LOGDIRECTORY/$host/cloud-init-output.log - find_etcd_bad_cert_errors $LOGDIRECTORY/$host/cse/cluster-provision.log $LOGDIRECTORY/$host/etcd_status.log - done -} - -# == MAIN == - -if [ "$#" -eq 0 ]; then - echo "" - echo "Usage:" - echo " $0 ./KubernetesLogs_%Y-%m-%d-%H-%M-%S-%3N" - echo "" - echo "ARTIFACTSURL can be overridden" - exit 1 -fi - -LOGDIRECTORY=$1 -ERRFILE="$LOGDIRECTORY/ISSUES.txt" - -ARTIFACTSURL="${ARTIFACTSURL:-https://raw.githubusercontent.com/msazurestackworkloads/azurestack-gallery/master}" -download_scripts $ARTIFACTSURL - -source scripts/detectors.sh $ERRFILE - -find_dvm_issues -find_node_issues diff --git a/diagnosis/getkuberneteslogs.sh b/diagnosis/getkuberneteslogs.sh index 21b07b4..e31c52d 100755 --- a/diagnosis/getkuberneteslogs.sh +++ b/diagnosis/getkuberneteslogs.sh @@ -1,85 +1,167 @@ #!/bin/bash -function restore_ssh_config +validateKeys() { - # Restore only if previously backed up - if [[ -v SSH_CONFIG_BAK ]]; then - if [ -f $SSH_CONFIG_BAK ]; then - rm ~/.ssh/config - mv $SSH_CONFIG_BAK ~/.ssh/config - fi + host=$1 + flags=$2 + + ssh ${flags} ${USER}@${host} "exit" + + if [ $? -ne 0 ]; then + echo "[$(date +%Y%m%d%H%M%S)][ERR] Error connecting to host ${host}" + exit 1 + fi +} + +validateResourceGroup() +{ + LOCATION=$(az group show -n ${RESOURCE_GROUP} --query location --output tsv) + if [ $? -ne 0 ]; then + echo "[$(date +%Y%m%d%H%M%S)][ERR] Specified resource group ${RESOURCE_GROUP} not found in current subscription." + exit 1 fi +} + +checkRequirements() +{ + if ! command -v az &> /dev/null; then + echo "[$(date +%Y%m%d%H%M%S)][ERR] azure-cli not available, please install and configure following this indications: https://docs.microsoft.com/azure-stack/user/azure-stack-version-profiles-azurecli2" + exit 1 + fi +} + +copyLogsToSADirectory() +{ + az vm list -g ${RESOURCE_GROUP} --show-details --query "[*].{host:name,akse:tags.aksEngineVersion}" --output table | grep 'k8s-' > ${SA_DIR}/akse-version.txt + + cp ${LOGFILEFOLDER}/k8s-*.zip ${SA_DIR} + cp ${LOGFILEFOLDER}/vmd-*.zip ${SA_DIR} + cp ${LOGFILEFOLDER}/cluster-snapshot.zip ${SA_DIR} + cp ${LOGFILEFOLDER}/resources/* ${SA_DIR} + + if [ -n "$API_MODEL" ] + then + cp ${API_MODEL} ${SA_DIR} + fi +} + +deleteSADirectory() +{ + rm -rf ${LOGFILEFOLDER}/data +} + +createSADirectories() +{ + local SA_DIR_DATE=$(echo $NOW | head -c 8) + local SA_DIR_HOUR=$(echo $NOW | tail -c 7 | head -c 2) + local SA_DIR_MIN=$(echo $NOW | tail -c 5 | head -c 2) + SA_CONTAINER_DIR="data/d=${SA_DIR_DATE}/h=${SA_DIR_HOUR}/m=${SA_DIR_MIN}" + SA_DIR="${LOGFILEFOLDER}/${SA_CONTAINER_DIR}" + mkdir -p ${SA_DIR} +} + +ensureResourceGroup() +{ + SA_RESOURCE_GROUP="KubernetesLogs" - # Restore only if previously backed up - if [[ -v SSH_KEY_BAK ]]; then - if [ -f $SSH_KEY_BAK ]; then - rm ~/.ssh/id_rsa - mv $SSH_KEY_BAK ~/.ssh/id_rsa - # Remove if empty - if [ -a ~/.ssh/id_rsa -a ! -s ~/.ssh/id_rsa ]; then - rm ~/.ssh/id_rsa - fi - fi + echo "[$(date +%Y%m%d%H%M%S)][INFO] Ensuring resource group: ${SA_RESOURCE_GROUP}" + az group create -n ${SA_RESOURCE_GROUP} -l ${LOCATION} 1> /dev/null + if [ $? -ne 0 ]; then + echo "[$(date +%Y%m%d%H%M%S)][ERR] Error ensuring resource group ${SA_RESOURCE_GROUP}" + exit 1 fi } -# Restorey SSH config file always, even if the script ends with an error -trap restore_ssh_config EXIT +ensureStorageAccount() +{ + SA_NAME="diagnostics" + + echo "[$(date +%Y%m%d%H%M%S)][INFO] Ensuring storage account: ${SA_NAME}" + az storage account create --name ${SA_NAME} --resource-group ${SA_RESOURCE_GROUP} --location ${LOCATION} --sku Premium_LRS --https-only true 1> /dev/null + if [ $? -ne 0 ]; then + echo "[$(date +%Y%m%d%H%M%S)][ERR] Error ensuring storage account ${SA_NAME}" + exit 1 + fi +} -function download_scripts +ensureStorageAccountContainer() { - ARTIFACTSURL=$1 - mkdir -p $SCRIPTSFOLDER + SA_CONTAINER="kuberneteslogs" - echo "[$(date +%Y%m%d%H%M%S)][INFO] Pulling dependencies from this repo: $ARTIFACTSURL" + echo "$(date +%Y%m%d%H%M%S)][INFO] Ensuring storage account container: ${SA_CONTAINER}" + az storage container create --name ${SA_CONTAINER} --account-name ${SA_NAME} + if [ $? -ne 0 ]; then + echo "$(date +%Y%m%d%H%M%S)][ERR] Error ensuring storage account container ${SA_CONTAINER}" + exit 1 + fi +} + +uploadLogs() +{ + echo "$(date +%Y%m%d%H%M%S)][INFO] Uploading log files to container: ${SA_CONTAINER}" + az storage blob upload-batch -d ${SA_CONTAINER} -s ${SA_DIR} --destination-path ${SA_CONTAINER_DIR} --account-name ${SA_NAME} + if [ $? -ne 0 ]; then + echo "$(date +%Y%m%d%H%M%S)][ERR] Error uploading log files to container ${SA_CONTAINER}" + exit 1 + fi +} + +processHost() +{ + host=$1 - for script in common detectors collectlogs collectlogsdvm hosts - do - if [ -f $SCRIPTSFOLDER/$script.sh ]; then - echo "[$(date +%Y%m%d%H%M%S)][INFO] Dependency '$script.sh' already in local file system" - fi - - curl -fs $ARTIFACTSURL/diagnosis/$script.sh -o $SCRIPTSFOLDER/$script.sh - - if [ ! -f $SCRIPTSFOLDER/$script.sh ]; then - echo "[$(date +%Y%m%d%H%M%S)][ERROR] Required script not available. URL: $ARTIFACTSURL/diagnosis/$script.sh" - echo "[$(date +%Y%m%d%H%M%S)][ERROR] You may be running an older version. Download the latest script from github: https://aka.ms/AzsK8sLogCollectorScript" - exit 1 - fi - done + echo "[$(date +%Y%m%d%H%M%S)][INFO] Processing host ${host}" + scp ${SCP_FLAGS} -o ProxyCommand="${PROXY_CMD}" collectlogs.sh ${USER}@${host}:/home/${USER}/collectlogs.sh + ssh ${SSH_FLAGS} -o ProxyCommand="${PROXY_CMD}" ${USER}@${host} "sudo chmod 744 collectlogs.sh; ./collectlogs.sh ${NAMESPACES};" + scp ${SCP_FLAGS} -o ProxyCommand="${PROXY_CMD}" ${USER}@${host}:/home/${USER}/${host}.zip ${LOGFILEFOLDER}/${host}.zip + ssh ${SSH_FLAGS} -o ProxyCommand="${PROXY_CMD}" ${USER}@${host} "rm -f collectlogs.sh ${host}.zip" } -function printUsage +processDvmHost() { + host=$1 + + DVM_NAME=$(az vm list -g ${RESOURCE_GROUP} --show-details --query "[*].{Name:name,ip:privateIps}" --output tsv | grep 'vmd-' | cut -f 1 ) + + echo "[$(date +%Y%m%d%H%M%S)][INFO] Processing dvm-host ${host}" + scp ${SCP_FLAGS} collectlogs.sh ${USER}@${host}:/home/${USER}/collectlogs.sh + ssh ${SSH_FLAGS} ${USER}@${host} "sudo chmod 744 collectlogs.sh; ./collectlogs.sh ${NAMESPACES};" + scp ${SCP_FLAGS} ${USER}@${host}:/home/${USER}/${DVM_NAME}.zip ${LOGFILEFOLDER}/${DVM_NAME}.zip + ssh ${SSH_FLAGS} ${USER}@${host} "rm -f collectlogs.sh ${DVM_NAME}.zip" +} + +printUsage() +{ + echo "$0 collects diagnostics from Kubernetes clusters provisioned by AKS Engine" echo "" echo "Usage:" - echo " $0 -i id_rsa -m 192.168.102.34 -u azureuser -n default -n monitoring --disable-host-key-checking" - echo " $0 --identity-file id_rsa --user azureuser --vmd-host 192.168.102.32" - echo " $0 --identity-file id_rsa --master-host 192.168.102.34 --user azureuser --vmd-host 192.168.102.32" - echo " $0 --identity-file id_rsa --master-host 192.168.102.34 --user azureuser --vmd-host 192.168.102.32" + echo " $0 [flags]" + echo "" + echo "Flags:" + echo " -u, --user The administrator username for the cluster VMs" + echo " -i, --identity-file RSA private key tied to the public key used to create the Kubernetes cluster (usually named 'id_rsa')" + echo " -g, --resource-group Kubernetes cluster resource group" + echo " --api-model AKS Engine Kubernetes cluster definition json file" + echo " --upload-logs Persists retrieved logs in an Azure Stack storage account" + echo " --disable-host-key-checking Sets SSH's StrictHostKeyChecking option to \"no\" while the script executes. Only use in a safe environment." + echo " -h, --help Print script usage" echo "" - echo "Options:" - echo " -u, --user User name associated to the identifity-file" - echo " -i, --identity-file RSA private key tied to the public key used to create the Kubernetes cluster (usually named 'id_rsa')" - echo " -m, --master-host A master node's public IP or FQDN (host name starts with 'k8s-master-')" - echo " -d, --vmd-host The DVM's public IP or FQDN (host name starts with 'vmd-')" - echo " -n, --user-namespace Collect logs for containers in the passed namespace (kube-system logs are always collected)" - echo " --all-namespaces Collect logs for all containers. Overrides the user-namespace flag" - echo " --disable-host-key-checking Sets SSH StrictHostKeyChecking option to \"no\" while the script executes. Use only when building automation in a save environment." - echo " -h, --help Print the command usage" + echo "Examples:" + echo " $0 -u azureuser -i ~/.ssh/id_rsa -g k8s-rg --disable-host-key-checking" + echo " $0 -u azureuser -i ~/.ssh/id_rsa -g k8s-rg -n default -n monitoring" + echo " $0 -u azureuser -i ~/.ssh/id_rsa -g k8s-rg --upload-logs --api-model clusterDefinition.json" + echo " $0 -u azureuser -i ~/.ssh/id_rsa -g k8s-rg --upload-logs" + exit 1 } - if [ "$#" -eq 0 ] then printUsage fi NAMESPACES="kube-system" -ALLNAMESPACES=1 -# Revert once CI passes the new flag => STRICT_HOST_KEY_CHECKING="ask" -STRICT_HOST_KEY_CHECKING="no" +UPLOAD_LOGS="false" # Handle named parameters while [[ "$#" -gt 0 ]] @@ -89,28 +171,24 @@ do IDENTITYFILE="$2" shift 2 ;; - -m|--master-host) - MASTER_HOST="$2" - shift 2 - ;; - -d|--vmd-host) - DVM_HOST="$2" - shift 2 - ;; -u|--user) USER="$2" shift 2 ;; - -n|--user-namespace) - NAMESPACES="$NAMESPACES $2" + -g|--resource-group) + RESOURCE_GROUP="$2" + shift 2 + ;; + --api-model) + API_MODEL="$2" shift 2 ;; - --all-namespaces) - ALLNAMESPACES=0 + --upload-logs) + UPLOAD_LOGS="true" shift ;; --disable-host-key-checking) - STRICT_HOST_KEY_CHECKING="no" + KNOWN_HOSTS_OPTIONS='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR' shift ;; -h|--help) @@ -118,7 +196,7 @@ do ;; *) echo "" - echo "[ERR] Incorrect option $1" + echo "[ERR] Unexpected flag $1" printUsage ;; esac @@ -139,135 +217,102 @@ then printUsage fi -if [ -z "$DVM_HOST" -a -z "$MASTER_HOST" ] +if [ ! -f $IDENTITYFILE ] then echo "" - echo "[ERR] Either --vmd-host or --master-host should be provided" + echo "[ERR] identity-file $IDENTITYFILE not found" printUsage + exit 1 +else + cat $IDENTITYFILE | grep -q "BEGIN \(RSA\|OPENSSH\) PRIVATE KEY" \ + || { echo "Provided identity file $IDENTITYFILE is not a RSA Private Key file."; echo "A RSA private key starts with '-----BEGIN [RSA|OPENSSH] PRIVATE KEY-----''"; exit 1; } fi -if [ ! -f $IDENTITYFILE ] +if [ -z "$RESOURCE_GROUP" ] then echo "" - echo "[ERR] identity-file not found at $IDENTITYFILE" + echo "[ERR] --resource-group is required" printUsage exit 1 -else - cat $IDENTITYFILE | grep -q "BEGIN \(RSA\|OPENSSH\) PRIVATE KEY" \ - || { echo "The identity file $IDENTITYFILE is not a RSA Private Key file."; echo "A RSA private key file starts with '-----BEGIN [RSA|OPENSSH] PRIVATE KEY-----''"; exit 1; } fi -test $ALLNAMESPACES -eq 0 && unset NAMESPACES +if [ -n "$API_MODEL" ] +then + if [ -n "$(grep -e secret -e "BEGIN CERTIFICATE" -e "BEGIN RSA PRIVATE KEY" $API_MODEL)" ] || [ -n "$(grep -Po '"secret": *\K"[^"]*"' $API_MODEL | sed -e 's/^"//' -e 's/"$//')" ] + then + echo "[ERR] --api-model contains sensitive information (secrets or certificates); please remove it before running the tool" + exit 1 + fi +fi # Print user input echo "" -echo "user: $USER" -echo "identity-file: $IDENTITYFILE" -echo "master-host: $MASTER_HOST" -echo "vmd-host: $DVM_HOST" -echo "namespaces: ${NAMESPACES:-all}" +echo "user: $USER" +echo "identity-file: $IDENTITYFILE" +echo "resource-group: $RESOURCE_GROUP" +echo "upload-logs: $UPLOAD_LOGS" echo "" NOW=`date +%Y%m%d%H%M%S` -CURRENTDATE=$(date +"%Y-%m-%d-%H-%M-%S-%3N") -LOGFILEFOLDER="./KubernetesLogs_$CURRENTDATE" -SCRIPTSFOLDER="$LOGFILEFOLDER/scripts" -mkdir -p $LOGFILEFOLDER/scripts -mkdir -p ~/.ssh - -# Download scripts from github -ARTIFACTSURL="${ARTIFACTSURL:-https://raw.githubusercontent.com/msazurestackworkloads/azurestack-gallery/master}" -download_scripts $ARTIFACTSURL - -# Backup .ssh/config -SSH_CONFIG_BAK=~/.ssh/config.$NOW -if [ ! -f ~/.ssh/config ]; then touch ~/.ssh/config; fi -mv ~/.ssh/config $SSH_CONFIG_BAK; - -# Backup .ssh/id_rsa -SSH_KEY_BAK=~/.ssh/id_rsa.$NOW -if [ ! -f ~/.ssh/id_rsa ]; then touch ~/.ssh/id_rsa; fi -mv ~/.ssh/id_rsa $SSH_KEY_BAK; -cp $IDENTITYFILE ~/.ssh/id_rsa - -echo "Host *" >> ~/.ssh/config -echo " StrictHostKeyChecking $STRICT_HOST_KEY_CHECKING" >> ~/.ssh/config -echo " UserKnownHostsFile /dev/null" >> ~/.ssh/config -echo " LogLevel ERROR" >> ~/.ssh/config - -echo "[$(date +%Y%m%d%H%M%S)][INFO] Testing SSH keys" -TEST_HOST="${MASTER_HOST:-$DVM_HOST}" -ssh -q $USER@$TEST_HOST "exit" +LOGFILEFOLDER="_output/${RESOURCE_GROUP}-${NOW}" +mkdir -p $LOGFILEFOLDER +SSH_FLAGS="-q -t -i ${IDENTITYFILE} ${KNOWN_HOSTS_OPTIONS}" +SCP_FLAGS="-q -i ${IDENTITYFILE} ${KNOWN_HOSTS_OPTIONS}" + +checkRequirements +validateResourceGroup + +# DVM +DVM_HOST=$(az network public-ip list -g ${RESOURCE_GROUP} --query "[*].{Name:name,ip:ipAddress}" --output tsv | grep 'vmd-' | head -n 1 | cut -f 2) + +if [ -n "$DVM_HOST" ] +then + echo "[$(date +%Y%m%d%H%M%S)][INFO] Checking connectivity with DVM host" + validateKeys ${DVM_HOST} "${SSH_FLAGS}" + + processDvmHost ${DVM_HOST} +fi + +# CLUSTER NODES +MASTER_IP=$(az network public-ip list -g ${RESOURCE_GROUP} --query "[*].{Name:name,ip:ipAddress}" --output tsv | grep 'k8s-master' | cut -f 2) if [ $? -ne 0 ]; then - echo "[$(date +%Y%m%d%H%M%S)][ERR] Error connecting to the server" - echo "[$(date +%Y%m%d%H%M%S)][ERR] Aborting log collection process" + echo "[$(date +%Y%m%d%H%M%S)][ERR] Error fetching the master nodes' load balancer IP" exit 1 fi -if [ -n "$MASTER_HOST" ] +echo "[$(date +%Y%m%d%H%M%S)][INFO] Checking connectivity with cluster nodes" +validateKeys ${MASTER_IP} "${SSH_FLAGS}" + +if [ -n "$MASTER_IP" ] then - echo "[$(date +%Y%m%d%H%M%S)][INFO] About to collect cluster logs" - - echo "[$(date +%Y%m%d%H%M%S)][INFO] Looking for cluster hosts" - scp -q $SCRIPTSFOLDER/hosts.sh $USER@$MASTER_HOST:/home/$USER/hosts.sh - ssh -tq $USER@$MASTER_HOST "sudo chmod 744 hosts.sh; ./hosts.sh $NOW" - scp -q $USER@$MASTER_HOST:"/home/$USER/cluster-info.$NOW" $LOGFILEFOLDER/cluster-info.tar.gz - ssh -tq $USER@$MASTER_HOST "sudo rm -f cluster-info.$NOW hosts.sh" - tar -xzf $LOGFILEFOLDER/cluster-info.tar.gz -C $LOGFILEFOLDER - rm $LOGFILEFOLDER/cluster-info.tar.gz + scp ${SCP_FLAGS} hosts.sh ${USER}@${MASTER_IP}:/home/${USER}/hosts.sh + ssh ${SSH_FLAGS} ${USER}@${MASTER_IP} "sudo chmod 744 hosts.sh; ./hosts.sh" + scp ${SCP_FLAGS} ${USER}@${MASTER_IP}:/home/${USER}/cluster-snapshot.zip ${LOGFILEFOLDER}/cluster-snapshot.zip + ssh ${SSH_FLAGS} ${USER}@${MASTER_IP} "sudo rm -f cluster-snapshot.zip hosts.sh" - # Configure SSH bastion host. Technically only needed for worker nodes. - for host in $(cat $LOGFILEFOLDER/host.list) - do - # https://en.wikibooks.org/wiki/OpenSSH/Cookbook/Proxies_and_Jump_Hosts#Passing_Through_One_or_More_Gateways_Using_ProxyJump - echo "Host $host" >> ~/.ssh/config - echo " ProxyJump $USER@$MASTER_HOST" >> ~/.ssh/config - done + CLUSTER_NODES=$(az vm list -g ${RESOURCE_GROUP} --show-details --query "[*].{Name:name,ip:privateIps}" --output tsv | grep 'k8s-' | cut -f 1) + PROXY_CMD="ssh -i ${IDENTITYFILE} ${KNOWN_HOSTS_OPTIONS} ${USER}@${MASTER_IP} -W %h:%p" - for host in $(cat $LOGFILEFOLDER/host.list) + for host in ${CLUSTER_NODES} do - echo "[$(date +%Y%m%d%H%M%S)][INFO] Processing host $host" - - echo "[$(date +%Y%m%d%H%M%S)][INFO] Uploading scripts" - scp -q -r $SCRIPTSFOLDER/*.sh $USER@$host:/home/$USER/ - ssh -q -t $USER@$host "sudo chmod 744 common.sh detectors.sh collectlogs.sh; ./collectlogs.sh $NAMESPACES;" - - echo "[$(date +%Y%m%d%H%M%S)][INFO] Downloading logs" - scp -q $USER@$host:"/home/$USER/kube_logs.tar.gz" $LOGFILEFOLDER/kube_logs.tar.gz - tar -xzf $LOGFILEFOLDER/kube_logs.tar.gz -C $LOGFILEFOLDER - rm $LOGFILEFOLDER/kube_logs.tar.gz - - # Removing temp files from node - ssh -q -t $USER@$host "rm -f common.sh detectors.sh collectlogs.sh collectlogsdvm.sh kube_logs.tar.gz" + processHost ${host} done - - rm $LOGFILEFOLDER/host.list fi -if [ -n "$DVM_HOST" ] -then - echo "[$(date +%Y%m%d%H%M%S)][INFO] About to collect VMD logs" - - echo "[$(date +%Y%m%d%H%M%S)][INFO] Uploading scripts" - scp -q -r $SCRIPTSFOLDER/*.sh $USER@$DVM_HOST:/home/$USER/ - ssh -q -t $USER@$DVM_HOST "sudo chmod 744 common.sh detectors.sh collectlogsdvm.sh; ./collectlogsdvm.sh;" - - echo "[$(date +%Y%m%d%H%M%S)][INFO] Downloading logs" - scp -q $USER@$DVM_HOST:"/home/$USER/dvm_logs.tar.gz" $LOGFILEFOLDER/dvm_logs.tar.gz - tar -xzf $LOGFILEFOLDER/dvm_logs.tar.gz -C $LOGFILEFOLDER - rm $LOGFILEFOLDER/dvm_logs.tar.gz - - echo "[$(date +%Y%m%d%H%M%S)][INFO] Removing temp files from DVM" - ssh -q -t $USER@$DVM_HOST "rm -f common.sh detectors.sh collectlogs.sh collectlogsdvm.sh dvm_logs.tar.gz" -fi +mkdir -p $LOGFILEFOLDER/resources +az network vnet list -g ${RESOURCE_GROUP} > ${LOGFILEFOLDER}/resources/vnets.json -# Aggregate ERRORS.txt -if [ `find $LOGFILEFOLDER -name ERRORS.txt | wc -w` -ne "0" ]; -then - echo "[$(date +%Y%m%d%H%M%S)][INFO] Known issues found. Details: $LOGFILEFOLDER/ALL_ERRORS.txt" - cat $LOGFILEFOLDER/*/ERRORS.txt &> $LOGFILEFOLDER/ALL_ERRORS.txt +# UPLOAD +if [ "$UPLOAD_LOGS" == "true" ]; then + echo "[$(date +%Y%m%d%H%M%S)][INFO] Processing logs" + createSADirectories + copyLogsToSADirectory + ensureResourceGroup + ensureStorageAccount + ensureStorageAccountContainer + uploadLogs + deleteSADirectory fi -echo "[$(date +%Y%m%d%H%M%S)][INFO] Done collecting Kubernetes logs" -echo "[$(date +%Y%m%d%H%M%S)][INFO] Logs can be found in this location: $LOGFILEFOLDER" +echo "[$(date +%Y%m%d%H%M%S)][INFO] Logs can be found here: $LOGFILEFOLDER" diff --git a/diagnosis/hosts.sh b/diagnosis/hosts.sh index ccc5980..3aea823 100644 --- a/diagnosis/hosts.sh +++ b/diagnosis/hosts.sh @@ -1,32 +1,12 @@ -#! /bin/bash +#!/bin/bash -### -# -# Find nodes on the network using the `ping` command instead of `kubectl get nodes`. -# This approach should still give us back results even if the Kubernetes API is having problems. -# -### +TMP=$(mktemp -d) +WD="cluster-snapshot" +LOGDIRECTORY=${TMP}/${WD} +mkdir -p ${LOGDIRECTORY} -WD=$1 -mkdir -p $WD +echo "[$(date +%Y%m%d%H%M%S)][INFO] Collecting cluster snapshot" +kubectl cluster-info dump --output-directory ${TMP} &> /dev/null +cp ${TMP}/*.json ${TMP}/kube-system/*.json ${LOGDIRECTORY} -HOSTLIST=$WD/host.list -rm -f $HOSTLIST - -echo "[$(date +%Y%m%d%H%M%S)][INFO] Searching for cluster nodes" -for ip in 10.240.0.{4..21}; -do - ping -c 1 -W 1 $ip | grep "64 bytes" | cut -d " " -f 4 | cut -d ":" -f 1 >> $HOSTLIST -done -for ip in 10.240.255.{5..12}; -do - ping -c 1 -W 1 $ip | grep "64 bytes" | cut -d " " -f 4 | cut -d ":" -f 1 >> $HOSTLIST -done - -echo "[$(date +%Y%m%d%H%M%S)][INFO] Dumping cluster-info" -kubectl cluster-info &> $WD/cluster-info.log -kubectl cluster-info dump &> $WD/cluster-info-dump.log -kubectl get events -n kube-system &> $WD/kube-system.events - -cd $WD/ && tar -czf ../cluster-info.$WD . && cd -rm -rf $WD +(cd $TMP && zip -q -r ~/${WD}.zip ${WD}) diff --git a/diagnosis/k8sCollectLogsCi.sh b/diagnosis/k8sCollectLogsCi.sh new file mode 100644 index 0000000..2b20def --- /dev/null +++ b/diagnosis/k8sCollectLogsCi.sh @@ -0,0 +1,231 @@ +#!/bin/bash -x + +validateKeys() +{ + host=$1 + flags=$2 + + ssh ${flags} ${USER}@${host} "exit" + + if [ $? -ne 0 ]; then + echo "[$(date +%Y%m%d%H%M%S)][ERR] Error connecting to host ${host}" + exit 1 + fi +} + +processHost() +{ + host=$1 + + echo "[$(date +%Y%m%d%H%M%S)][INFO] Processing host ${host}" + scp ${SCP_FLAGS} -o ProxyCommand="${PROXY_CMD}" collectlogs.sh ${USER}@${host}:/home/${USER}/collectlogs.sh + ssh ${SSH_FLAGS} -o ProxyCommand="${PROXY_CMD}" ${USER}@${host} "sudo chmod 744 collectlogs.sh; ./collectlogs.sh ${NAMESPACES};" + scp ${SCP_FLAGS} -o ProxyCommand="${PROXY_CMD}" ${USER}@${host}:/home/${USER}/${host}.zip ${LOGFILEFOLDER}/${host}.zip + ssh ${SSH_FLAGS} -o ProxyCommand="${PROXY_CMD}" ${USER}@${host} "rm -f collectlogs.sh ${host}.zip" +} + +processDvmHost() +{ + host=$1 + dvm_name=$2 + echo "[$(date +%Y%m%d%H%M%S)][INFO] Processing dvm-host ${host}" + scp ${SCP_FLAGS} collectlogs.sh ${USER}@${host}:/home/${USER}/collectlogs.sh + ssh ${SSH_FLAGS} ${USER}@${host} "sudo chmod 744 collectlogs.sh; ./collectlogs.sh ${NAMESPACES};" + scp ${SCP_FLAGS} ${USER}@${host}:/home/${USER}/${dvm_name}.zip ${LOGFILEFOLDER}/${dvm_name}.zip + ssh ${SSH_FLAGS} ${USER}@${host} "rm -f collectlogs.sh ${dvm_name}.zip" +} + +printUsage() +{ + echo "$0 collects diagnostics from Kubernetes clusters provisioned by AKS Engine" + echo "" + echo "Usage:" + echo " $0 [flags]" + echo "" + echo "Flags:" + echo " -u, --user The administrator username for the cluster VMs" + echo " -i, --identity-file RSA private key tied to the public key used to create the Kubernetes cluster (usually named 'id_rsa')" + echo " -g, --resource-group Kubernetes cluster resource group" + echo " -n, --user-namespace Collect logs from containers in the specified namespaces (kube-system logs are always collected)" + echo " --api-model AKS Engine Kubernetes cluster definition json file" + echo " --all-namespaces Collect logs from containers in all namespaces. It overrides --user-namespace" + echo " --disable-host-key-checking Sets SSH's StrictHostKeyChecking option to \"no\" while the script executes. Only use in a safe environment." + echo " -h, --help Print script usage" + echo "" + echo "Examples:" + echo " $0 -u azureuser -i ~/.ssh/id_rsa -g k8s-rg --disable-host-key-checking" + echo " $0 -u azureuser -i ~/.ssh/id_rsa -g k8s-rg -n default -n monitoring" + + exit 1 +} + +if [ "$#" -eq 0 ] +then + printUsage +fi + +NAMESPACES="kube-system" +ALLNAMESPACES=1 + +# Handle named parameters +while [[ "$#" -gt 0 ]] +do + case $1 in + -i|--identity-file) + IDENTITYFILE="$2" + shift 2 + ;; + -u|--user) + USER="$2" + shift 2 + ;; + -g|--resource-group) + RESOURCE_GROUP="$2" + shift 2 + ;; + --dvm-ip) + DVM_HOST="$2" + shift 2 + ;; + --dvm-name) + DVM_NAME="$2" + shift 2 + ;; + --master-ip) + MASTER_IP="$2" + shift 2 + ;; + --aksupstreamflag) + AKS_UPSTREAM_FLAG=true + shift + ;; + -n|--user-namespace) + NAMESPACES="$NAMESPACES $2" + shift 2 + ;; + --all-namespaces) + ALLNAMESPACES=0 + shift + ;; + --disable-host-key-checking) + KNOWN_HOSTS_OPTIONS='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR' + shift + ;; + -h|--help) + printUsage + ;; + *) + echo "" + echo "[ERR] Unexpected flag $1" + printUsage + ;; + esac +done + +# Validate input +if [ -z "$USER" ] +then + echo "" + echo "[ERR] --user is required" + printUsage +fi + +if [ -z "$IDENTITYFILE" ] +then + echo "" + echo "[ERR] --identity-file is required" + printUsage +fi + +if [ ! -f $IDENTITYFILE ] +then + echo "" + echo "[ERR] identity-file $IDENTITYFILE not found" + printUsage + exit 1 +else + cat $IDENTITYFILE | grep -q "BEGIN \(RSA\|OPENSSH\) PRIVATE KEY" \ + || { echo "Provided identity file $IDENTITYFILE is not a RSA Private Key file."; echo "A RSA private key starts with '-----BEGIN [RSA|OPENSSH] PRIVATE KEY-----''"; exit 1; } +fi + +test $ALLNAMESPACES -eq 0 && unset NAMESPACES + +# Print user input +echo "" +echo "user: $USER" +echo "identity-file: $IDENTITYFILE" +echo "resource-group: $RESOURCE_GROUP" +echo "namespaces: ${NAMESPACES:-all}" +echo "" + +NOW=`date +%Y%m%d%H%M%S` +LOGFILEFOLDER="_output/${RESOURCE_GROUP}-${NOW}" +mkdir -p $LOGFILEFOLDER + +SSH_FLAGS="-q -t -i ${IDENTITYFILE} ${KNOWN_HOSTS_OPTIONS}" +SCP_FLAGS="-q -i ${IDENTITYFILE} ${KNOWN_HOSTS_OPTIONS}" + + +if [ -n "$DVM_HOST" ] +then + echo "[$(date +%Y%m%d%H%M%S)][INFO] Checking connectivity with DVM host" + validateKeys ${DVM_HOST} "${SSH_FLAGS}" + processDvmHost ${DVM_HOST} ${DVM_NAME} +fi + +if [ "$AKS_UPSTREAM_FLAG" = true ] +then + ssh ${SSH_FLAGS} ${USER}@${DVM_HOST} "cd /home/azureuser/src/github.com/Azure/aks-engine; sudo cp _output/*-ssh /home/azureuser/aksMasterIdentityFile; sudo chmod 744 /home/azureuser/aksMasterIdentityFile " + scp ${SCP_FLAGS} ${USER}@${DVM_HOST}:/home/${USER}/aksMasterIdentityFile ${LOGFILEFOLDER}/aksMasterIdentityFile + scp ${SCP_FLAGS} ${USER}@${DVM_HOST}:/home/${USER}/src/github.com/Azure/aks-engine/deploy_test_results ${LOGFILEFOLDER}/deploy_test_results + scp ${SCP_FLAGS} ${USER}@${DVM_HOST}:/home/${USER}/src/github.com/Azure/aks-engine/scale_* ${LOGFILEFOLDER}/ + scp ${SCP_FLAGS} ${USER}@${DVM_HOST}:/home/${USER}/src/github.com/Azure/aks-engine/upgrade_* ${LOGFILEFOLDER}/ + + if [ ! -f ${LOGFILEFOLDER}/aksMasterIdentityFile ] + then + echo "Not able to find kubernetes cluster identity file" + exit 1 + else + IDENTITYFILE=${LOGFILEFOLDER}/aksMasterIdentityFile + SSH_FLAGS="-q -t -i ${IDENTITYFILE} ${KNOWN_HOSTS_OPTIONS}" + SCP_FLAGS="-q -i ${IDENTITYFILE} ${KNOWN_HOSTS_OPTIONS}" + fi +fi + +if [ -n "$MASTER_IP" ] +then + echo "[$(date +%Y%m%d%H%M%S)][INFO] Checking connectivity with master node" + validateKeys ${MASTER_IP} "${SSH_FLAGS}" + + scp ${SCP_FLAGS} hosts.sh ${USER}@${MASTER_IP}:/home/${USER}/hosts.sh + ssh ${SSH_FLAGS} ${USER}@${MASTER_IP} "sudo chmod 744 hosts.sh; ./hosts.sh" + scp ${SCP_FLAGS} ${USER}@${MASTER_IP}:/home/${USER}/cluster-snapshot.zip ${LOGFILEFOLDER}/cluster-snapshot.zip + ssh ${SSH_FLAGS} ${USER}@${MASTER_IP} "kubectl get nodes -o json|jq -r '.items[].metadata.name' | grep -v k8s-master-* > cluster_nodes.txt" + scp ${SCP_FLAGS} ${USER}@${MASTER_IP}:/home/${USER}/cluster_nodes.txt ${LOGFILEFOLDER}/cluster-nodes.txt + ssh ${SSH_FLAGS} ${USER}@${MASTER_IP} "sudo rm -f cluster-snapshot.zip hosts.sh cluster_nodes.txt" + + if [ ! -f ${LOGFILEFOLDER}/cluster-nodes.txt ] + then + echo "Agent nodes not present" + else + PROXY_CMD="ssh -i ${IDENTITYFILE} ${KNOWN_HOSTS_OPTIONS} ${USER}@${MASTER_IP} -W %h:%p" + + INPUT_FILE=${LOGFILEFOLDER}/cluster-nodes.txt + CLUSTER_NODES=$(<$INPUT_FILE) + + for host in ${CLUSTER_NODES} + do + processHost ${host} + done + fi +fi + +echo "Removing master files copied" +if [ -f ${LOGFILEFOLDER}/aksMasterIdentityFile ] +then + echo "Removing master files copied" + rm ${LOGFILEFOLDER}/aksMasterIdentityFile +fi + +echo "[$(date +%Y%m%d%H%M%S)][INFO] Logs can be found here: $LOGFILEFOLDER" + diff --git a/diagnosis/scripts/release.sh b/diagnosis/scripts/release.sh new file mode 100755 index 0000000..c01f730 --- /dev/null +++ b/diagnosis/scripts/release.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +VERSION=$1 + +dos2unix *.sh +mkdir -p _out +tar -czf _out/diagnosis-v${VERSION}.tar.gz collectlogs.sh getkuberneteslogs.sh hosts.sh +zip -q _out/diagnosis-v${VERSION}.zip collectlogs.sh getkuberneteslogs.sh hosts.sh diff --git a/diagnosis/scripts/updateconfiguration.sh b/diagnosis/scripts/updateconfiguration.sh new file mode 100644 index 0000000..c02af39 --- /dev/null +++ b/diagnosis/scripts/updateconfiguration.sh @@ -0,0 +1,153 @@ +#!/bin/bash -e + +agentScript() { + cat < agent.sh +#!/bin/bash -e + +# cloud provider +sudo cp /etc/kubernetes/azure.json /etc/kubernetes/azure.bak +sudo jq '.cloudProviderRatelimit = true | .cloudProviderRateLimitQPS = 3 | .cloudProviderRateLimitBucket = 10 | .cloudProviderBackoffRetries = 1 | .cloudProviderBackoffDuration = 30' /etc/kubernetes/azure.bak | sudo dd status=none of=/etc/kubernetes/azure.json + +# kubelet +sudo cp /etc/default/kubelet /etc/default/kubelet.bak +sudo sed -i -e 's/--node-status-update-frequency=[0-9]*[a-z]/--node-status-update-frequency=1m/' /etc/default/kubelet + +# restart +sudo systemctl daemon-reload +sudo systemctl restart kubelet + +echo "=> azure.json updates" +sudo grep -E 'cloudProviderRatelimit"|cloudProviderRateLimitQPS"|cloudProviderRateLimitBucket"|cloudProviderBackoffRetries"|cloudProviderBackoffDuration"' /etc/kubernetes/azure.json +echo "=> kubelet restarted" +systemctl status kubelet --no-pager -l +EOF +if [ ! -f agent.sh ]; then + echo "[ERR] Error generating script: agent.sh" + return 1 +fi +} + +masterScript() { + cat < master.sh +#!/bin/bash -e + +# cloud provider +sudo cp /etc/kubernetes/azure.json /etc/kubernetes/azure.bak +sudo jq '.cloudProviderRatelimit = false | .cloudProviderRateLimitQPS = 3 | .cloudProviderRateLimitBucket = 10 | .cloudProviderBackoffRetries = 1 | .cloudProviderBackoffDuration = 30' /etc/kubernetes/azure.bak | sudo dd status=none of=/etc/kubernetes/azure.json + +# kubelet +sudo cp /etc/default/kubelet /etc/default/kubelet.bak +sudo sed -i -e 's/--node-status-update-frequency=[0-9]*[a-z]/--node-status-update-frequency=1m/' /etc/default/kubelet + +# kube-controller-manager +sudo cp /etc/kubernetes/manifests/kube-controller-manager.yaml /etc/kubernetes/manifests/kube-controller-manager.bak +sudo sed -i -e 's/--route-reconciliation-period=[0-9]*[a-z]/route-reconciliation-period=1m/' /etc/kubernetes/manifests/kube-controller-manager.yaml +sudo sed -i -e 's/--node-monitor-grace-period=[0-9]*[a-z]/--node-monitor-grace-period=5m/' /etc/kubernetes/manifests/kube-controller-manager.yaml +sudo sed -i -e 's/--pod-eviction-timeout=[0-9]*[a-z]/--pod-eviction-timeout=5m/' /etc/kubernetes/manifests/kube-controller-manager.yaml + +# restart +sudo systemctl daemon-reload +sudo systemctl restart kubelet + +echo "=> controller-manager updates" +grep -o -E 'route-reconciliation-period=[0-9a-zA-Z]*' /etc/kubernetes/manifests/kube-controller-manager.yaml +grep -o -E 'node-monitor-grace-period=[0-9a-zA-Z]*' /etc/kubernetes/manifests/kube-controller-manager.yaml +grep -o -E 'pod-eviction-timeout=[0-9a-zA-Z]*' /etc/kubernetes/manifests/kube-controller-manager.yaml +echo "=> azure.json updates" +sudo grep -E 'cloudProviderRatelimit"|cloudProviderRateLimitQPS"|cloudProviderRateLimitBucket"|cloudProviderBackoffRetries"|cloudProviderBackoffDuration"' /etc/kubernetes/azure.json +echo "=> kubelet restarted" +systemctl status kubelet --no-pager -l +EOF +if [ ! -f master.sh ]; then + echo "[ERR] Error generating script: master.sh" + return 1 +fi +} + +processHost() { + HOST=$1 + SCRIPT=$2 + + if [[ "$HOST" == "$HOSTNAME" ]]; then + sudo chmod +x ${SCRIPT}; + ./${SCRIPT}; + else + KNOWN_HOSTS_OPTIONS='-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR' + PROXY_CMD="ssh ${KNOWN_HOSTS_OPTIONS} ${USER}@${HOSTNAME} -W %h:%p" + SSH_FLAGS="-q -t ${KNOWN_HOSTS_OPTIONS}" + SCP_FLAGS="-q ${KNOWN_HOSTS_OPTIONS}" + + scp ${SCP_FLAGS} -o ProxyCommand="${PROXY_CMD}" ${SCRIPT} ${USER}@${HOST}:${SCRIPT} + ssh ${SSH_FLAGS} -o ProxyCommand="${PROXY_CMD}" ${USER}@${HOST} "sudo chmod +x ${SCRIPT}; ./${SCRIPT}; rm ${SCRIPT};" + fi +} + +printUsage() +{ + echo "$0 updates Kubernetes clusters configuration on agent and master nodes." + echo "Usage:" + echo " $0 [flags]" + echo "" + echo "Flags:" + echo " --agents Update configuration on all Kubernetes agent nodes" + echo " --masters Update configuration on all Kubernetes master nodes" + echo "" + echo "Examples:" + echo " $0 --agents" + echo " $0 --masters" + echo " $0 --masters --agents" + exit 1 +} + +if [ "$#" -eq 0 ] +then + printUsage +fi + +DOMASTERS=1 +DOAGENTS=1 +while [[ "$#" -gt 0 ]] +do + case $1 in + --masters) + DOMASTERS=0 + shift + ;; + --agents) + DOAGENTS=0 + shift + ;; + -h|--help) + printUsage + ;; + *) + echo "" + echo "[ERR] Unexpected flag $1" + printUsage + ;; + esac +done + +## NODES +if [ $DOAGENTS -eq 0 ]; then + agentScript + AGENTS=$(kubectl get nodes -o custom-columns=CONTAINER:.metadata.name | tail -n +2 | grep -v k8s-master | xargs) + for AGENT in ${AGENTS}; do + echo "" + echo "==> PROCESSING AGENT $AGENT" + processHost ${AGENT} agent.sh + done + rm agent.sh +fi + +# MASTERS +if [ $DOMASTERS -eq 0 ]; then + masterScript + MASTERS=$(kubectl get nodes -o custom-columns=CONTAINER:.metadata.name | tail -n +2 | grep k8s-master | xargs) + for MASTER in ${MASTERS}; do + echo "" + echo "==> PROCESSING MASTER $MASTER" + processHost ${MASTER} master.sh + done + rm master.sh +fi diff --git a/kubernetes/Kubernetes Cluster using AKS Engine - Standalone (free) Use Terms.docx b/kubernetes/Kubernetes Cluster using AKS Engine - Standalone (free) Use Terms.docx new file mode 100644 index 0000000..3c306f2 Binary files /dev/null and b/kubernetes/Kubernetes Cluster using AKS Engine - Standalone (free) Use Terms.docx differ diff --git a/kubernetes/Microsoft AKS Engine for Azure Stack Preview - Pre-Release or Evaluation Use Terms.docx b/kubernetes/Microsoft AKS Engine for Azure Stack Preview - Pre-Release or Evaluation Use Terms.docx deleted file mode 100644 index bafd37d..0000000 Binary files a/kubernetes/Microsoft AKS Engine for Azure Stack Preview - Pre-Release or Evaluation Use Terms.docx and /dev/null differ diff --git a/kubernetes/docs/troubleshooting.md b/kubernetes/docs/troubleshooting.md index 115bdde..71319c8 100644 --- a/kubernetes/docs/troubleshooting.md +++ b/kubernetes/docs/troubleshooting.md @@ -17,6 +17,7 @@ If the Azure Stack portal does not provide enough information for you to trouble | ERR_AKSE_GENERATE | 11 | Failure thrown by AKS Engine's `generate` operation | | ERR_AKSE_DEPLOY | 12 | Failure thrown by AKS Engine's `deploy` operation | | ERR_TEMPLATE_DOWNLOAD | 13 | Failure downloading AKS-Engine cluster definition | +| ERR_TEMPLATE_GENERATION | 15 | Failure to generate cluster definition | | ERR_CACERT_INSTALL | 20 | Failure handling CA certificate | | ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT | 26 | Timeout waiting for Microsoft's GPG key download | | ERR_METADATA_ENDPOINT | 30 | Non-successful call to Azure Stack's metadata endpoint | diff --git a/kubernetes/template/DeploymentTemplates/MainTemplate.json b/kubernetes/template/DeploymentTemplates/MainTemplate.json index ece113d..d5174c8 100644 --- a/kubernetes/template/DeploymentTemplates/MainTemplate.json +++ b/kubernetes/template/DeploymentTemplates/MainTemplate.json @@ -6,7 +6,7 @@ "type": "string", "defaultValue": "azureuser", "metadata": { - "description": "User name for the Linux Virtual Machines that are part of the Kubernetes cluster and DVM." + "description": "User name for the Linux virtual machines that are part of the Kubernetes cluster and DVM." } }, "sshPublicKey": { @@ -24,14 +24,14 @@ "agentPoolProfileCount": { "defaultValue": 3, "metadata": { - "description": "Kubernetes Node Pool Profile Count" + "description": "Kubernetes linux node pool profile count" }, "type": "int" }, "agentPoolProfileVMSize": { "defaultValue": "Standard_D2_v2", "metadata": { - "description": "The VMSize of Kubernetes node VMs" + "description": "The virtual machine size of the Kubernetes linux agent nodes" }, "type": "string" }, @@ -45,7 +45,7 @@ "masterPoolProfileVMSize": { "defaultValue": "Standard_D2_v2", "metadata": { - "description": "The VMSize of Kubernetes master VMs" + "description": "The virtual machine size of the Kubernetes master nodes" }, "type": "string" }, @@ -82,11 +82,18 @@ }, "kubernetesAzureCloudProviderVersion": { "type": "string", - "defaultValue": "1.12", + "defaultValue": "", "metadata": { "description": "This is the version for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." } }, + "kubernetesAzureCloudProviderRelease": { + "type": "string", + "defaultValue": "1.14", + "metadata": { + "description": "This is the release for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." + } + }, "_artifactsLocation": { "type": "string", "metadata": { @@ -103,7 +110,7 @@ }, "aksEngineReleaseVersion": { "type": "string", - "defaultValue": "v0.37.4", + "defaultValue": "v0.48.0", "metadata": { "description": "The version of AKS Engine to download" } @@ -129,56 +136,185 @@ "description": "The name of the file containing the cluster definition" } }, + "customVnetName": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Name of the custom vnet" + } + }, + "networkPlugin": { + "defaultValue": "kubenet", + "allowedValues": [ + "flannel", + "azure", + "kubenet" + ], + "metadata": { + "description": "Network plugin which will deployed in Kubernetes cluster" + }, + "type": "string" + }, + "networkPolicy": { + "defaultValue": "", + "allowedValues": [ + "", + "azure" + ], + "metadata": { + "description": "Network policy which will deployed in Kubernetes cluster" + }, + "type": "string" + }, + "windowsAdminUsername": { + "defaultValue": "azureuser", + "metadata": { + "description": "User name for the Windows virtual machines that are part of the Kubernetes cluster." + }, + "type": "string" + }, + "windowsAdminPassword": { + "defaultValue": "", + "metadata": { + "description": "User password for the Windows virtual machines that are part of the Kubernetes cluster." + }, + "type": "securestring" + }, + "customWindowsPackageURL": { + "defaultValue": "", + "metadata": { + "description": "Custom Windows K8s zip location which will be used to deploy(kubelet, kubeproxy) on Windows node." + }, + "type": "string" + }, + "windowsAgentPoolProfileCount": { + "defaultValue": "0", + "metadata": { + "description": "Kubernetes Windows node pool profile count" + }, + "type": "string" + }, + "windowsAgentPoolProfileVMSize": { + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The virtual machine size of the Kubernetes Windows nodes" + }, + "type": "string" + }, + "availabilityProfile": { + "defaultValue": "AvailabilitySet", + "allowedValues": [ + "AvailabilitySet", + "VirtualMachineScaleSets" + ], + "metadata": { + "description": "Availability profile that nodes in the Kubernetes cluster will be deployed with" + }, + "type": "string" + }, + "masterSubnetName": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Name of the master subnet" + } + }, + "agentSubnetName": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Name of the agent subnet" + } + }, + "firstConsecutiveStaticIP": { + "type": "string", + "defaultValue": "10.100.0.5", + "metadata": { + "description": "First consecutive static IP" + } + }, "nodeDistro": { - "defaultValue": "ubuntu", + "defaultValue": "aks-ubuntu-16.04", "allowedValues": [ "ubuntu", - "aks" + "aks-ubuntu-16.04" ], "metadata": { "description": "Node distro to be used to deploy Kubernetes on Azure Stack." }, "type": "string" }, - "kubernetesImageBase": { + "enableTillerAddOn": { + "type": "string", + "defaultValue": "false", + "metadata": { + "description": "Flag to enable Tiller addon" + } + }, + "containerRuntime": { + "defaultValue": "docker", + "allowedValues": [ + "docker", + "containerd" + ], + "metadata": { + "description": "Container runtime to deploy on each cluster node." + }, + "type": "string" + }, + "localAKSeBinaryURL": { "type": "string", - "defaultValue": "mcr.microsoft.com/k8s/azurestack/core/", + "defaultValue": "", "metadata": { - "description": "The base for Kubernetes images" + "description": "Azure Stack blob url to download AKS engine from a blob store in disconnected environment." + } + }, + "dvmImagePublisher": { + "type": "string", + "defaultValue": "microsoft-aks", + "metadata": { + "description": "Azure VM image publisher to be used for the DVM" + } + }, + "dvmImageOffer": { + "type": "string", + "defaultValue": "aks", + "metadata": { + "description": "Azure VM image offer to be used for the DVM" + } + }, + "dvmImageSku": { + "type": "string", + "defaultValue": "aks-engine-ubuntu-1604-202003", + "metadata": { + "description": "Azure VM image sku to be used for the DVM" + } + }, + "dvmImageVersion": { + "type": "string", + "defaultValue": "2020.03.19", + "metadata": { + "description": "Azure VM image version to be used for the DVM" } } }, "variables": { - "resourceGroupName": "[resourceGroup().name]", - "dnsNameForPublicIP": "[toLower(concat('vmd-dns', parameters('masterProfileDnsPrefix')))]", - "location": "[resourceGroup().location]", - "imagePublisher": "Canonical", - "imageOffer": "UbuntuServer", - "imageSku": "16.04-LTS", - "imageVersion": "latest", - "vmSize": "Standard_D2_v2", - "OSDiskName": "osdisk", - "nicName": "[concat('vmd-vnic', uniqueString(resourceGroup().id))]", - "addressPrefix": "10.0.0.0/24", - "subnetName": "mySubnet", - "subnetPrefix": "10.0.0.0/24", - "storageAccountName": "[concat('vmdsa', uniquestring(resourceGroup().id))]", - "storageAccountType": "Standard_LRS", - "publicIPAddressName": "[concat('vmd-publicIP', uniqueString(resourceGroup().id))]", - "publicIPAddressType": "Static", - "vmStorageAccountContainerName": "vhds", - "vmName": "[concat('vmd-', uniqueString(resourceGroup().id))]", - "virtualNetworkName": "[concat('vmd-vnet-', uniqueString(resourceGroup().id))]", - "vnetID": "[resourceId('Microsoft.Network/virtualNetworks',variables('virtualNetworkName'))]", - "subnetRef": "[concat(variables('vnetID'),'/subnets/',variables('subnetName'))]", - "networkSecurityGroupName": "[tolower(concat('vmd-nsg',uniquestring(resourceGroup().id)))]", - "sshKeyPath": "[concat('/home/',parameters('linuxAdminUsername'),'/.ssh/authorized_keys')]", - "tenantSubscriptionId": "[subscription().subscriptionId]", - "scriptName": "script", - "singleQuote": "'", - "scriptParameters": "[concat('IDENTITY_SYSTEM=','\"',parameters('identitySystem'),'\"',' NODE_DISTRO=','\"',parameters('nodeDistro'),'\"',' RESOURCE_GROUP_NAME=','\"',variables('resourceGroupName'),'\"',' PUBLICIP_DNS=','\"',variables('dnsNameForPublicIP'),'\"' ,' TENANT_ID=','\"',subscription().tenantId,'\"' ,' TENANT_SUBSCRIPTION_ID=','\"',variables('tenantSubscriptionId'),'\"',' ADMIN_USERNAME=','\"',parameters('linuxAdminUsername'),'\"',' MASTER_DNS_PREFIX=','\"',parameters('masterProfileDnsPrefix'),'\"' ,' AGENT_COUNT=','\"',parameters('agentPoolProfileCount'),'\"' ,' AGENT_SIZE=','\"',parameters('agentPoolProfileVMSize'),'\"' ,' MASTER_COUNT=','\"',parameters('masterPoolProfileCount'),'\"',' MASTER_SIZE=','\"',parameters('masterPoolProfileVMSize'),'\"' ,' SPN_CLIENT_ID=','\"',parameters('servicePrincipalClientId'),'\"' ,' SPN_CLIENT_SECRET=','\"',parameters('servicePrincipalClientSecret'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_VERSION=','\"',parameters('kubernetesAzureCloudProviderVersion'),'\"' ,' REGION_NAME=','\"',variables('location'),'\"' ,' SSH_PUBLICKEY=','\"',parameters('sshPublicKey'),'\"' ,' STORAGE_PROFILE=','\"',parameters('storageProfile'),'\"')]" + "apiVersionDeployments": "2015-11-01" }, "resources": [ + { + "apiVersion": "[variables('apiVersionDeployments')]", + "name": "pid-b8a190d4-9ecc-4bd5-a9ba-3cc60c6e0c1f", + "properties": { + "mode": "Incremental", + "template": { + "$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "resources": [] + } + }, + "type": "Microsoft.Resources/deployments" + }, { "apiVersion": "2015-01-01", "name": "dvmdeployment", @@ -223,12 +359,36 @@ "kubernetesAzureCloudProviderVersion": { "value": "[parameters('kubernetesAzureCloudProviderVersion')]" }, + "kubernetesAzureCloudProviderRelease": { + "value": "[parameters('kubernetesAzureCloudProviderRelease')]" + }, "identitySystem": { "value": "[parameters('identitySystem')]" }, "nodeDistro": { "value": "[parameters('nodeDistro')]" }, + "customVnetName": { + "value": "[parameters('customVnetName')]" + }, + "networkPlugin": { + "value": "[parameters('networkPlugin')]" + }, + "networkPolicy": { + "value": "[parameters('networkPolicy')]" + }, + "availabilityProfile": { + "value": "[parameters('availabilityProfile')]" + }, + "masterSubnetName": { + "value": "[parameters('masterSubnetName')]" + }, + "agentSubnetName": { + "value": "[parameters('agentSubnetName')]" + }, + "firstConsecutiveStaticIP": { + "value": "[parameters('firstConsecutiveStaticIP')]" + }, "aksEngineBaseURL": { "value": "[parameters('aksEngineBaseURL')]" }, @@ -244,8 +404,41 @@ "clusterDefinitionFileName": { "value": "[parameters('clusterDefinitionFileName')]" }, - "kubernetesImageBase": { - "value": "[parameters('kubernetesImageBase')]" + "windowsAgentPoolProfileCount": { + "value": "[parameters('windowsAgentPoolProfileCount')]" + }, + "windowsAgentPoolProfileVMSize": { + "value": "[parameters('windowsAgentPoolProfileVMSize')]" + }, + "windowsAdminUsername": { + "value": "[parameters('windowsAdminUsername')]" + }, + "windowsAdminPassword": { + "value": "[parameters('windowsAdminPassword')]" + }, + "customWindowsPackageURL": { + "value": "[parameters('customWindowsPackageURL')]" + }, + "enableTillerAddOn": { + "value": "[parameters('enableTillerAddOn')]" + }, + "containerRuntime": { + "value": "[parameters('containerRuntime')]" + }, + "localAKSeBinaryURL": { + "value": "[parameters('localAKSeBinaryURL')]" + }, + "dvmImagePublisher": { + "value": "[parameters('dvmImagePublisher')]" + }, + "dvmImageOffer": { + "value": "[parameters('dvmImageOffer')]" + }, + "dvmImageSku": { + "value": "[parameters('dvmImageSku')]" + }, + "dvmImageVersion": { + "value": "[parameters('dvmImageVersion')]" } } } @@ -257,4 +450,4 @@ "value": "[reference('dvmdeployment').outputs.dvmPublicIpFqdn.value]" } } -} \ No newline at end of file +} diff --git a/kubernetes/template/DeploymentTemplates/azuredeploy.json b/kubernetes/template/DeploymentTemplates/azuredeploy.json index 19c0768..6b4ad08 100644 --- a/kubernetes/template/DeploymentTemplates/azuredeploy.json +++ b/kubernetes/template/DeploymentTemplates/azuredeploy.json @@ -6,7 +6,7 @@ "type": "string", "defaultValue": "azureuser", "metadata": { - "description": "User name for the Linux Virtual Machines that are part of the Kubernetes cluster and DVM." + "description": "User name for the Linux virtual machines that are part of the Kubernetes cluster and DVM." } }, "sshPublicKey": { @@ -24,28 +24,28 @@ "agentPoolProfileCount": { "defaultValue": 3, "metadata": { - "description": "Kubernetes Node Pool Profile Count" + "description": "Kubernetes linux node pool profile count" }, "type": "int" }, "agentPoolProfileVMSize": { "defaultValue": "Standard_D2_v2", "metadata": { - "description": "The VMSize of Kubernetes node VMs" + "description": "The virtual machine size of the Kubernetes linux agent nodes" }, "type": "string" }, "masterPoolProfileCount": { "defaultValue": 3, "metadata": { - "description": "Kubernetes Master Pool Profile Count" + "description": "Kubernetes master pool profile count" }, "type": "int" }, "masterPoolProfileVMSize": { "defaultValue": "Standard_D2_v2", "metadata": { - "description": "The VMSize of Kubernetes master VMs" + "description": "The virtual machine size of the Kubernetes master nodes" }, "type": "string" }, @@ -81,11 +81,18 @@ }, "kubernetesAzureCloudProviderVersion": { "type": "string", - "defaultValue": "1.12", + "defaultValue": "", "metadata": { "description": "This is the version for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." } }, + "kubernetesAzureCloudProviderRelease": { + "type": "string", + "defaultValue": "1.14", + "metadata": { + "description": "This is the release for the Kubernetes Azure cloud provider. We would use a custom Kubernetes build specifically for Azure Stack for each version." + } + }, "aksEngineBaseURL": { "type": "string", "defaultValue": "https://github.com/Azure/aks-engine/releases/download", @@ -95,7 +102,7 @@ }, "aksEngineReleaseVersion": { "type": "string", - "defaultValue": "v0.37.4", + "defaultValue": "v0.48.0", "metadata": { "description": "The version of AKS Engine to download" } @@ -121,33 +128,172 @@ "description": "The name of the file containing the cluster definition" } }, - "kubernetesImageBase": { + "customVnetName": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Name of the custom vnet" + } + }, + "masterSubnetName": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Name of the master subnet" + } + }, + "agentSubnetName": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Name of the agent subnet" + } + }, + "firstConsecutiveStaticIP": { "type": "string", - "defaultValue": "mcr.microsoft.com/k8s/azurestack/core/", + "defaultValue": "10.100.0.5", "metadata": { - "description": "The base for Kubernetes images" + "description": "First Consective Static IP" } }, + "networkPlugin": { + "defaultValue": "kubenet", + "allowedValues": [ + "flannel", + "azure", + "kubenet" + ], + "metadata": { + "description": "Network plugin which will deployed in Kubernetes cluster" + }, + "type": "string" + }, + "networkPolicy": { + "defaultValue": "", + "allowedValues": [ + "", + "azure" + ], + "metadata": { + "description": "Network policy which will deployed in Kubernetes cluster" + }, + "type": "string" + }, + "availabilityProfile": { + "defaultValue": "AvailabilitySet", + "allowedValues": [ + "AvailabilitySet", + "VirtualMachineScaleSets" + ], + "metadata": { + "description": "Availability profile that nodes in the Kubernetes cluster will be deployed with" + }, + "type": "string" + }, + "windowsAgentPoolProfileCount": { + "defaultValue": "0", + "metadata": { + "description": "Kubernetes Windows node pool profile count" + }, + "type": "string" + }, + "windowsAgentPoolProfileVMSize": { + "defaultValue": "Standard_D2_v2", + "metadata": { + "description": "The virtual machine size of the Kubernetes Windows agent nodes" + }, + "type": "string" + }, + "windowsAdminUsername": { + "defaultValue": "azureuser", + "metadata": { + "description": "User name for the Windows virtual machines that are part of the Kubernetes cluster." + }, + "type": "string" + }, + "windowsAdminPassword": { + "defaultValue": "", + "metadata": { + "description": "Password for the Windows virtual machines that are part of the Kubernetes cluster." + }, + "type": "securestring" + }, + "customWindowsPackageURL": { + "defaultValue": "", + "metadata": { + "description": "Custom Windows K8s zip location which will be used to deploy(kubelet, kubeproxy) on Windows node." + }, + "type": "string" + }, "nodeDistro": { - "defaultValue": "ubuntu", + "defaultValue": "aks-ubuntu-16.04", "allowedValues": [ "ubuntu", - "aks" + "aks-ubuntu-16.04" ], "metadata": { "description": "Node distro to be used to deploy Kubernetes on Azure Stack." }, "type": "string" + }, + "enableTillerAddOn": { + "type": "string", + "defaultValue": "false", + "metadata": { + "description": "Flag to enable Tiller addon" + } + }, + "containerRuntime": { + "type": "string", + "defaultValue": "docker", + "allowedValues": [ + "docker", + "containerd" + ], + "metadata": { + "description": "Container runtime to deploy on each cluster node." + } + }, + "localAKSeBinaryURL": { + "type": "string", + "defaultValue": "", + "metadata": { + "description": "Azure Stack blob url to download AKS engine from a blob store in disconnected environment " + } + }, + "dvmImagePublisher": { + "type": "string", + "defaultValue": "microsoft-aks", + "metadata": { + "description": "Azure VM image publisher to be used for the DVM" + } + }, + "dvmImageOffer": { + "type": "string", + "defaultValue": "aks", + "metadata": { + "description": "Azure VM image offer to be used for the DVM" + } + }, + "dvmImageSku": { + "type": "string", + "defaultValue": "aks-engine-ubuntu-1604-202003", + "metadata": { + "description": "Azure VM image sku to be used for the DVM" + } + }, + "dvmImageVersion": { + "type": "string", + "defaultValue": "2020.03.19", + "metadata": { + "description": "Azure VM image version to be used for the DVM" + } } }, "variables": { "resourceGroupName": "[resourceGroup().name]", "dnsNameForPublicIP": "[toLower(concat('vmd-dns', parameters('masterProfileDnsPrefix')))]", "location": "[resourceGroup().location]", - "imagePublisher": "Canonical", - "imageOffer": "UbuntuServer", - "imageSku": "16.04-LTS", - "imageVersion": "latest", "vmSize": "Standard_D2_v2", "OSDiskName": "osdisk", "nicName": "[concat('vmd-vnic', uniqueString(resourceGroup().id))]", @@ -168,7 +314,7 @@ "tenantSubscriptionId": "[subscription().subscriptionId]", "scriptName": "script", "singleQuote": "'", - "scriptParameters": "[concat('IDENTITY_SYSTEM=','\"',parameters('identitySystem'),'\"',' NODE_DISTRO=','\"',parameters('nodeDistro'),'\"',' RESOURCE_GROUP_NAME=','\"',variables('resourceGroupName'),'\"',' PUBLICIP_DNS=','\"',variables('dnsNameForPublicIP'),'\"' ,' TENANT_ID=','\"',subscription().tenantId,'\"' ,' TENANT_SUBSCRIPTION_ID=','\"',variables('tenantSubscriptionId'),'\"',' ADMIN_USERNAME=','\"',parameters('linuxAdminUsername'),'\"',' MASTER_DNS_PREFIX=','\"',parameters('masterProfileDnsPrefix'),'\"' ,' AGENT_COUNT=','\"',parameters('agentPoolProfileCount'),'\"' ,' AGENT_SIZE=','\"',parameters('agentPoolProfileVMSize'),'\"' ,' MASTER_COUNT=','\"',parameters('masterPoolProfileCount'),'\"',' MASTER_SIZE=','\"',parameters('masterPoolProfileVMSize'),'\"' ,' SPN_CLIENT_ID=','\"',parameters('servicePrincipalClientId'),'\"' ,' SPN_CLIENT_SECRET=','\"',parameters('servicePrincipalClientSecret'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_VERSION=','\"',parameters('kubernetesAzureCloudProviderVersion'),'\"' ,' REGION_NAME=','\"',variables('location'),'\"' ,' SSH_PUBLICKEY=','\"',parameters('sshPublicKey'),'\"' ,' STORAGE_PROFILE=','\"',parameters('storageProfile'),'\"',' AKSE_BASE_URL=','\"',parameters('aksEngineBaseURL'), '\"', ' AKSE_RELEASE_VERSION=','\"',parameters('aksEngineReleaseVersion'),'\"',' GALLERY_REPO=','\"',parameters('galleryRepository'),'\"',' GALLERY_BRANCH=','\"',parameters('galleryBranch'),'\"',' DEFINITION_TEMPLATE_NAME=','\"',parameters('clusterDefinitionFileName'),'\"',' K8S_IMAGE_BASE=','\"',parameters('kubernetesImageBase'),'\"')]" + "scriptParameters": "[concat('IDENTITY_SYSTEM=','\"',parameters('identitySystem'),'\"',' ENABLE_TILLER=','\"',parameters('enableTillerAddOn'),'\"' ,' CONTAINER_RUNTIME=','\"',parameters('containerRuntime'),'\"' ,' WINDOWS_CUSTOM_PACKAGE=','\"',parameters('customWindowsPackageURL'),'\"' ,' WINDOWS_AGENT_COUNT=','\"',parameters('windowsAgentPoolProfileCount'),'\"' ,' WINDOWS_AGENT_SIZE=','\"',parameters('windowsAgentPoolProfileVMSize'),'\"',' WINDOWS_ADMIN_USERNAME=','\"',parameters('windowsAdminUsername'),'\"',' WINDOWS_ADMIN_PASSWORD=','\"',parameters('windowsAdminPassword'),'\"',' NETWORK_PLUGIN=','\"',parameters('networkPlugin'),'\"',' NETWORK_POLICY=','\"',parameters('networkPolicy'),'\"',' AVAILABILITY_PROFILE=','\"',parameters('availabilityProfile'),'\"',' FIRST_CONSECUTIVE_STATIC_IP=','\"',parameters('firstConsecutiveStaticIP'),'\"',' AGENT_SUBNET_NAME=','\"',parameters('agentSubnetName'),'\"',' MASTER_SUBNET_NAME=','\"',parameters('masterSubnetName'),'\"',' CUSTOM_VNET_NAME=','\"',parameters('customVnetName'),'\"',' NODE_DISTRO=','\"',parameters('nodeDistro'),'\"',' RESOURCE_GROUP_NAME=','\"',variables('resourceGroupName'),'\"',' PUBLICIP_DNS=','\"',variables('dnsNameForPublicIP'),'\"' ,' TENANT_ID=','\"',subscription().tenantId,'\"' ,' TENANT_SUBSCRIPTION_ID=','\"',variables('tenantSubscriptionId'),'\"',' ADMIN_USERNAME=','\"',parameters('linuxAdminUsername'),'\"',' MASTER_DNS_PREFIX=','\"',parameters('masterProfileDnsPrefix'),'\"' ,' AGENT_COUNT=','\"',parameters('agentPoolProfileCount'),'\"' ,' AGENT_SIZE=','\"',parameters('agentPoolProfileVMSize'),'\"' ,' MASTER_COUNT=','\"',parameters('masterPoolProfileCount'),'\"',' MASTER_SIZE=','\"',parameters('masterPoolProfileVMSize'),'\"' ,' SPN_CLIENT_ID=','\"',parameters('servicePrincipalClientId'),'\"' ,' SPN_CLIENT_SECRET=','\"',parameters('servicePrincipalClientSecret'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_RELEASE=','\"',parameters('kubernetesAzureCloudProviderRelease'),'\"' ,' K8S_AZURE_CLOUDPROVIDER_VERSION=','\"',parameters('kubernetesAzureCloudProviderVersion'),'\"' ,' REGION_NAME=','\"',variables('location'),'\"' ,' SSH_PUBLICKEY=','\"',parameters('sshPublicKey'),'\"' ,' STORAGE_PROFILE=','\"',parameters('storageProfile'),'\"',' AKSE_BASE_URL=','\"',parameters('aksEngineBaseURL'), '\"', ' AKSE_RELEASE_VERSION=','\"',parameters('aksEngineReleaseVersion'),'\"',' GALLERY_REPO=','\"',parameters('galleryRepository'),'\"',' GALLERY_BRANCH=','\"',parameters('galleryBranch'),'\"',' DEFINITION_TEMPLATE_NAME=','\"',parameters('clusterDefinitionFileName'),'\"',' DISCONNECTED_AKS_ENGINE_URL=','\"',parameters('localAKSeBinaryURL'),'\"')]" }, "resources": [ { @@ -281,7 +427,7 @@ "vmSize": "[variables('vmSize')]" }, "osProfile": { - "customData": "[base64(concat('#cloud-config\n\nwrite_files:\n- path: \"/opt/azure/containers/script.sh\"\n permissions: \"0744\"\n encoding: gzip\n owner: \"root\"\n content: !!binary |\n H4sIAAAAAAAA/7w763LbttL/+RR7aE6ddErJdo875ySlW0WiXU1kSUeSm6ZxPgaiIBk1CTAAaFtp+u7fALyIN0nuxdEPj0nsHYvF7gI8+Fd7Tmh7jsQN2Ngw3MnE64xnXn84nXUGA2/Wv3RHVzPnv3AAMxJiFksgVEgUBISugOOPMeF4ASiSECH/Fq2wSIi8nrpeb/RmOBh1es7xERzAOSJBzDEs2D0NGFooAp3XU9ulK0IxzAlFnJTQ3fFg9NY5Pikg+ynnDeKhgAWOArYGFmGOJGFUk5i5l+NBZ1aU4tv9UkgcRgGSWJPodrruJDeGc1LUImR3CrXbAR9zSZbEz7Au3Vmn15l1PHfYG4/6w5nz7VGDBvIGQ4glWiCJANNFxAiV6QT0vctRzx04/y4izmMSJOKO+xCyBQ4gFuo5FpgDoVGcov869bqD0VXPm7gX/enMnTinTQKgT+AHLF4AxysiJOb57F+Ne8pw+eQXZ/8eEamwl4yrWbdXWEIcLZDEIBn4LIwCLLFxkM/ihTt0J52Z6xwf75vHFaZqBnFhJjWdy6l3Mb7wXrtv88nMhTv5botwl8TnTLClPBRwMb6AW7zOJz0VLzeUO/y5PxkNL93hzDktiimwlCVbYXpHOKMhprJKZDwZnfcHrnN6sotAxNmSBHiDPBhd9Idep9NzTov+KRkEbAWEqv86nV4z5xS5dz51Tv+9Fbt3Pm1G73S7o6vhzJtevXJOT5vE9n0WUwkLvERxIEHEc+FzEum5MYxlTH31L/gsCLAvvWQlKiYeogsvn0dh/G4AABzAVDK1ehChgB+IBJ8tsB5yf+nPvO6o5zrWD/qF/kOW8A6sfAxsiuEI3r9Uy4dqCPUL2MoL8B0OwCZgdmMhWTjVcgJ+kJgKJeMSkQAv4J7Imw3rAm1Tk1uSDfNHkeUxLXg+YbSVENIsNtSNPwzjALqJnWBjJ8VEAAru0Vp8A/gOU6WzCg4i5UQXIhEaUcCcM25IjqL9JtcWNYyDgwPjAAC+F3EYIr4+00/qN1CcWSyjWMJ8DRHHEaY6wujljOhCO1FiAbmO8DNXsf8G7hGnhK6+AUKXDBiHO8znTODnrYRTu8Tq+whxFAJFIXbMY/Nsto4yB23Bs59RQBZwh4IYC0AcvwAb6/WLE172vX7KWdpEPyvO3wAOBM6Zf9/WjBqYtlot82yUKIr9GwZCckJXrTIGxzLmVJwNGcXft7OnZAw/+Fg7fTq6ec5wQ8RvxVkXBZmPEQoChdk0ipYimQDpKcl969nzdG34SGAwrWMTyMazbfw8Edm0nqlZeQ7vXM7fA5hg/f7ji5M/csiXLzdI9zWkN4jT97uRSA2pT5dsN9LXz6GK1MPzeLUdCwvk67WwzS21Qyj/I0tQkVItJCG1NxIJN0gAZRQ+Yc5gvpZYtKC/VP6jkNR6jJAQWKj9GYuEgGRA8T0EzNcQj3LSKYu5j+Fc4Q9RiFvbvevEPOthIQlN+G9DeZx7dVEaF5MwpdZAEqeOVTDXLqJ1WjCsDCFT6zAOgnzCQIS2TOuf8NDUpliHFY6FCtyeH8QqVfAWeEkoUfRz/9Wh+l9gC7COd0VoDGY3oQIbKolWh9bxYYNqRCrFcBjJdQs6aRSEGxRFmCpdbhRuiCiJ4gDpfYtIAb8JvTFRialMY3Iel4+LsT68UxJbJzvdErosIlhA55PaH6cS+bfAGZPF9E95morcKIo4izhRr7TZml3uywYcTEWsJnAjrsinrrzRXSapbRFSaVbTKrFp59eriTuddbqvvcloNPNUytw/73dVBjkdXU26rjfuzH5yzPYd4u2AzNv3CK0wle1ugUErwuF+ej13OsuoxYK31ZIO2uIGcdz2kV2UuI0+iQKDls+luXVnZ9EaUnxYchbCofVItQ6VZXZD50IfQiKBH8Fjye8GzClvFHsE3fP+8MKdjCeqMvnAIkyFCODh9Oi/YBP6eMlsylTKbS8JXWEecULlhwbb/jmBXjyKfwHBfJzq2lJ/Q/Hc0v+E2lVh9ihdBS+onBRedsX1N+MHMMkq9Pk6DV3dQT/NTyPGJUzc/12509nU63a8V1fD3sB12lj6bSGCtiIqqgtLraSEgN73/xK+CZ9BYgw2Ag1cKE7gDNoLfNemcRAkOXMvLdogFEhpIFTsvWf8Vr0Uh6qE1Bnjrc4R1L4PHAf4DlGpdxWRhWWFo7NbwrEvGV8bWT3ooVuB82ioy9Zf+2Nv2Ll0HRPdChvrEtW29NDEHbidqev97E6m/dHQDgiNH2wULr77t1kmcDUZOGaC9EphXE0G7UYa6duMaUsi3lp9SsNFzAOwbY4lX8Np9p+9wAFaw/ER2HaIHmxJQgzfHYE9UDuwvQR7BFZREPj8OatJqv2ZjcOEtwvCwY6g1Z6nWahEHOyHJTRKmMWzVkWB9sZoBVIb4wxG3c6sPxo6erAAXcokMqYZ9J6sosBTN5PWOotYsliljiqtirAv8SJPBM0tdNwq4IuKHLuSCf0n7z1pB7iRMhIv2m2O7lsrIm/ieSwwzzITn4Vt66IzGLiTt97EHY82T68mnWH3p/ZtPMecYrWpZe2pdi+v+mbpK9G2eu55f9hXEm66X2o2/pIf5U5UVKbsRLUWm6EZNYjhmK194hUmvQFwz9RvWmKZhXSV+Dq33D/oCg3S7XWIQm7855XLAiBewH49idB9ApUr6+jX2qZa5jd/VzWVzTTpVNzTuqPhef/iaqJXz7YsrL580R0iAZoHWM1ZhOQNWOM3vUrESBWsbL91OzUQ2yZhy0x2ns4dI0lve5n0xYTaRpeECwn+Dfb1bpJ2awLm36bbDeIYEFe+ZhzAedYik+gW0yS7VPBq1zoUSW03RwIvjHtEpLdk3EOR9BS5TX6eVDhL3ejN0+hFdLtqa7b5KxTJdkCEFIX3PvJvsB5B3L8hdzgdPMu3WTg5++r4JSzYZqrV1n74ptBNVTuqKj7ZUttDi3eYw4sA4wi+1c8LRnHRfJIjKnyitnYUSTvtFWcG/VMWUoZZYeklNLZUL1dqUPcuIwla+VYhY1Ixj2DhHB/pR0UxoeYljTCnLcOonTa2U2FbLJYbCrr7pHzIeibwRzgGK6X5vGTC+mzmQ2riwLZ9RpdkpVIyG+VjWUtdxd51dtRSG02NqOYtzaSsmibwGVYcR2C7YP7fs3dv3PcvWl8///zsHXbfc976+rm1WdrvwPohba3Ce/jqK/CRbCL51Vcw5xjdqm3gOsduBs6HkxYuARt/zG1Vi3jp3MScpiFGu2EgcAki9bOj/GUaiLTP5TOUJKbuA/ZjmZxNFY1mEVB7XFsk6jfM03bnTefj73lvSiR338wlreOXiTRaS8c6eakFZbF0rG+T5pm4IUs9C+V/vrRzZmawbcoym9gc+ywMdbfaXoP1+49/VNwLf8zdq8GJ/nEvsTa2LI2Wg8hf8qRH6H9tWj9em5mzCfPlLmd7ip9xAEIiLg2jHB6n6qUKj4Wcwc/bcVlS+QLujlqnrWOzgj2JKVXI6dkEEi8ArGf3NwyF5HkVeLoWEoe6Vc/DLMV4FlMUKq96blZFs/+hX1WOzuQSdCcWS8xFdfTJuPYu+0PvaupOVI77Arb/rDJojdCFO5x5+qBuFxVNaAPaTGXa/3WnKAUqCrRGpFjJ7iFSBG2kUymBd9GpgFbJbSss6pruqEHKJMsl2M7ZK4NuI6Qqu92WLxWBVTL9njuc9Wdvvenb6cy93EHJqoBWKb3+z9TTiW9ybD2ejH7u99xJZtsCZWsPaBPl/mXnIpn3naqWQauELjvTmTt5jMtbRdAtZHrDqTeeuOf9X7bSsmqgW2g9Yv1YBdAqleGo53q9/nQ22eMLYBVAq1TGV68G/W5/rMTdY54i6FYy5//rDXfSsUqgtT3BvVDLaV+UU3QKoHUqaXP3YjK6Gu+iZjWA1rae8dDrDvoqjvV7O6VqCt0F7KnbnbjbnbARe/qTlxjstfv2z/OejSZqXaS3SXYt9ApoldLMHXb2668p5aBbaEyvXk27k/5Yx81mglYz6Bfb5LtM5WFUPuH+bhSuCSW9Yd1R12eB3YDFC6NzNfvJu3RnP416jukHqm7wBPY5lqZRCcveYPTGnTimbml7aFEzVJXZC7Cqr76YcfuUYg53iBM0D/CTmviXmcqCBjrSOKb1eyn0tNuliNYqBpSWvl+Foqj1h2mkzpjfwMt7sCGiaIVViltGtkqMTaN4IJOFm/rFPtOqMGpnN/ra2Y0+8QOKiH2HuSCMOidHx6f20bF9dFybu0dx1CvPehRodZLq/rQ1HNT9rEKqaKvdO0fFrE8c60pm2h3rNmZ6Omd+0+nP9EVFtY2Mhr2pc3JU4VbssVk1eBDYZ6qilAxQELD75GJYUldJpiq8OQnIJ9wyXxpp2Vsj8nQFZunWr/GINly59q6cSNUunlZJ9jcXj3XUtP2A6DO/hb64hqlPsChwycr0b4/gGL47OgK4NiL0ANfGbx/h2tDnIdcK2tZtnohxaes4AddGIOZ21vO8NgRbynvEsR1xFmEuCRa2qvYZhWtjQXhIVxyujZo+lWvUTzgTmwO0am2047b1WlvPr1/GyZrmwjSM/Egw670bxra+ubNpz+cHta3fBKNbMXQR1ogmw8gon8/uOsB8MsOWztWrNUHzPRm1RGsHTAKQ2FyXn691f9BsupRTVrN8Bf0plzJJzkuSm5hVVbssjGK9nBu8JUFpmYaRbULOh+SwMTlDTI8cj48euXN9KJugNm7otpyVvYez/B594mxGBuiNR5NZZ+B8qCB8ht8+wmFLLXgUuOlGfQif4QHxlfhQz8RK5HQiVnpjGkbSvqzkeOA4YHZ651Oz1MmsbILpVWt9S1tE2FeO0GTmTexJDzS2ZZSLpTCLd0BmnISQvAYRL5fkIblwnSnR6c76P6tic+J2Z6PJ201+s8VwKFa6SJLepwzYitCaGeEzCLwAG8OhaCvmX1vtdsHIedv2ieX4YCzJ1hndyrI4yVuBzC+wHleY1lq4qcckn0sEe33F0Ac020IwfE42RNtGfAUVz665ulo2G9q+vhKva59x8l1DuqiuJgNwasiHJVaF9LvUGYDDVhY1FY3CyGFZ1lrTpqGPU5Y3RMpQmagLKsYcL8mDYlPDrDAr9GNKzZmdDIiQnCnqBYScbglR34gcMxakuOLd0fuW/vbi0Cm2dctCbTq1xa7tIyjfhVPyCTsFpL+kbiPtRGtnv85lY2XKFjt6jfOdqFt82EE1U7QAXtG03HyvNuPLtPVNr4w0WoSEXgnM9YGGU8WssCl1hcC0fi+9+MPcwUiIm1YUzwPiv8ZrbeJbvO4hiRTTEpmqapt2BFjFh32LuBxYL7G8YQunSKGqXLHdBlb5scRMYH5HfDzmhPokQkE++7pZ0l9olYro2zklrbkSePrqURyTvkyFX0KgwrNxn61t9unrfaYlC2VYuU4Ox5xmKhUBXv9nmp+S7GvKlwVg3L/BQnIkWb4kiu8mSYnjWAUWDcwLXf1K634/u83tta4+Uy686IdohV/l/DdUD+HaONu+ZemqwXjsBwp76OzaGcv1XPpVZr3CooWPM+ubcesp04TNnSlVjyK6qKXuyU3B5DvRtAgEN7s5ZVTvX2bf0l4b9qqx065GbBQRO0lPthtPw8Xyxg51+CgHID2oa3hM7+oNJz2eJwGl5ECNJMHCJotqqCkMpsu7IToooGwV2mkvZctiVqDFTx41y+ZGd1N1qr9frs7HznP3/APalmn8fwAAAP//z1WRNJw9AAA='))]", + "customData": "[base64(concat('#cloud-config\n\nwrite_files:\n- path: \"/opt/azure/containers/script.sh\"\n permissions: \"0744\"\n encoding: gzip\n owner: \"root\"\n content: !!binary |\n H4sIAAAAAAAA/+x9e3fbtpPo//wUU5qnTrqhZDlJtz+lSleRaEfHsqSV5KRpnKtAJCShpgiWAG0rj+9+DwC+RUpKtu7ePXf9RyoSg8FgMBjMC+zRD/U58epzxFZg4ntNs8bjWXs0nfUGk2m7359Ne5fW8Gra+hekf0cwJWtMQw7EYxy5LvGWEOC/QhJgB5DPwUf2DVpiptBdTKxZd/h20B+2u63GCRT+juAMETcMMDj0znMpcgS+9sXEtLwl8TDMiYcCksNmjfrDd63GaRFXFpsdUZZiOmbgYN+lG6A+DhAn1JM4p9blqN+eZql8+k0Ucrz2XcSxxNYbvGn3e91Z+9waTGed4dVgOnvT7l9ZrcYzhe0V5St4SzyH3jFAngN94oX3gJbY43CL3BADYfAJBzRP3rk1sMbtaW84aDWe59ZjhcHBCxS6HJBPYE0d7IJNQ9cBj3KYY1hiT0wZOxJlp92xxskqt06Li5LOeE1vxWQ7bbBxwMmC2PE8L61pu9uetmfWoDsa9gbT1tOT3SvBVxjWmCMHcQTYc3xKPB6JXG92Oexa/dazLfnI45qHxFXsH/WieYZMPIcMB0A8P4ww/jGZdfrDq+5sbJ33JlNr3Hp+spM69Alsl4YOBHhJGMdBshmuRl3B/mQvlG+GO0S4wLOggdgE5hJzCH0HcQycgk3Xvos51o4SKY6W02o1GvvENl69jOBKPJeT2fnofHZhvUtkNyHz9OcK4i6JHVBGF/yYwfnoHG7wJhHsiLyEedbgTW88HFxag2nreZZMhjnPcQ17tySg3hp7vIhkNB6e9fpW6/npLgR+QBfExWnn/vC8N5i1293W86eZjpyCS5dAPPGr3e6Wjxx17p5NWs+fVfbunk3Ku7c7auNOrl61nj8vI9u2aejxZNuxcM7sgPhybbSjoyPtCAB+ZeF6jYLNS/kk/vp0yYCG3A85zDfgB9jHnhRoKSlCGwj6XHyLXeAbHz+ygoAGT+AOBR7xlk+AeAsKNIBbHMwpw49raqR6bqhffRSgNXhojVt6Q3853fjx3Gvw6A1yiaM0DQMU4CaYWIoGVmOZd/IpGdIk8lmM/ASwy3Ay+K91OVDJoLVaTX85VBPF9ooC4wHxlrV8jwDzMPDYywH18K/1+Em14XsbS35Grelz3HeNghv2soNcFztwR/iKeMDQGoNaClYTKBWQXBKXLmeSs48ea581sRw2Yhh0o6ED8bR4T5v4sSJZNx6JVXkM7+UifABogg7G5/9onn5NoF+8SDvebXV8q3j4YU9HstWx5y3oB9g34k+PodjxjVqaHSNihmztq6Yhn8+WmM+iYzxhSoC5OG9bRuOFVBwz5mLst4zTF8CVPmkZTxUqtiILDj/+WPgh2+Q/Um7EdjMeMfwXNMCIsD9+AQ5NKJPDLGgwEzS51L5hSZPj3yzBNG3qLchSbEETJW2xno1mAKbpUTN6MANs0/Uaew4DcyNYkTLiPRi/gYn/ghOAD4LqeYDRDXz5AtcJDFkIMCLBYqLhwwtxiqWiErErDDxoJC/FDslBSP6BkfIy1xovgzoskqYFkT8d6uGUnWqtrXtsh1wZW4fO/1o3/uNaFxMSa8h0tX4lfP+q7dJfY8yDDSzJLfbEmbYWCmsev/DC9RwHQBdqDLHscoPdYVihWwwrwgF5G1goZXqQ5hokOMUabGrVGudUf/kWES7Hhjnmdxh7eztJNdWJJsKpMJVwxN3/VlUl6bbXzowsZhG7yvZnsohid37LhhOa4enXFzt2QkZoXySSqH4lelScDHrMvUQqYxmr6S+ENG3LWDKTuxVxMSyk7Va/RUHdJfO62PB1AZe+Qj6vu4RxlnlvI3uFZQsK7BW5xVHjy7qDb+te6Lpw+vLHRm7Wcvscv80YQwF2sRBRupCeiyTvWMtv3KfpRvyqaUfQvqXEAR4gj9lE2OxiG0amXrRWTDuCs9CzhQwARzfYg0VA19IEbl9MhE1nUwfDHDHsaHkNkHAnz+Yr0ShND5+DnHxNTxVDLBONEy3WKQrbTBkbrTpf+/VIX0TE1mj4j2tqcyGUUaSitlojJop1gy/AMQZjaybwBZYB9sG0QP8/j96/tT40az89/vLoPbY+BEHtp8eGXtDzHk6k20a8DGX5CVAO/HAHxNOT71L+EdOiXVdn+ndpdmkTCjxkAcIOB3xPGJcGKeGwQgw86kmnFOYbjlkNegthQopOQsh9xBhmwl/ETCHgFDx8J7aUhDhI209oGNgYzkT/AVrj3eq+ixknnhq/qstharuDPDFhLlWzMoPVBm0kh5ick0Mxky614g4NgJFPibte+zs0v73C9s0Mec5M8HKGPgl1wpF9M4s3lpzwo8fwORUNKYsmA6MBH6Aggnk1ckZDz4Fjo3EsZuYjvoJjY/S2e6yWmonNhTgOgK9Qbr2VWOVEN4MZgy5XQCIucIl48RBCxShWgYu9JV/VQFrWsEK+8IPACYWTkHr50qHP+PpiWdxQeOfC8yIekZKVbni5hmqjRVuoRJNeqpjGIqGXUzg2To8VGhY6FNa3gpXGaflxJz15B2zBqXwsQuKUMqQsIsbR2peOU7BGEa3iDPnfTfj/9iaMeIrlPhT7jwZ4FkneLJW85KiW+++HeAdWb0AMemdLfjOSuD01wsXE8Nrnmxq0vYgtar+IuUgLao084oeukkbCGfzJqLDPPY49vnN7JIL+dZfV36H+Rgq6UEYToYwgoJRnY4JC0pDvB9QPiHiULCsXt3/WksYeCwPcSSll+wyszKxY6bQydlf7j6uxNZm2Oxez8XA4nXWs8bR31uu0p9ZsMrwad6zZqD193dITW/YOyUBvPUtRzcdrfS++rjWZxthCFghzF7l1tkIBrtvIzJJdR59YZoCaHXC9Sh2KxZVzVibqsXHgnCK9eRDFx5BRrrYPh46xGzBB/03rcdYbnFvj0bg3mLY+Uh97jLlw//zkX2AK6/dQykyP0lAYtMRb4sAPiMc/HraE/wUCkhlXD1+ywt/GlOZBPMh00L9xyCIT9gxYBM8IsjJ9zYLwp+1HSmNBp98D5mNbQIC9Qt4Ss5qyZ+59GnAYW/95ZU2mk1mnPXt1Nej2rVYdc7vOmFsXqFlxg4kdlUo0w46YtVDv1wb6LmQ6SJhsNLpUqK2I/9vpF92YWoP2YJq8qcfJlnqcbGG/IZ+YtzhghHqt05PGc/OkYZ40FEvtMHDzS1E5mHKEXyFGbHAwR8RlMuyzwsA2jOO1lheHceh5QrVGShoQgyYYj+5WFK3JY10rgE8kjqzdJKAlq0NheICJHuva23ZvKtMds4nVGQ66k9bpSQFR1tU3tuCBYZt6DpPNim6h01REjVNhvc1dofOFCtW1KCKyhUYcM3//X5En5t/0pxfwtseXIG01zHHAiq0PNmr3sjeYXU2s8aB9aTWh+s/Ig24hStOsu7BIRCloOZZJ74+dpGSwCNAtJBcTa3DeG1hpPrMCm1ECWo3t1bg96LyuJswoglajGgy71i5mGWWg1ejG1mi4e+1yoNWIrkbn43bXmr2xxpPecLCF06gE3cL5pt3rt1/1+r3puzjzWDXXEtAiul7XGkxF++TdZGpd7pisUQAtYrr4ZTKTulVlRUfj4Zte1xqXTNnYA3oo5rHVt9oT6xDMEWgR82V7MrXGh+wuIwtagaY7mMxGY+us93slLmMLtALXAVvVyIAWsQys6dvh+GI26l+d97bFLYslD1pENLp61e91eiNB8R4OZUEr0Zz9Z3cnORk0ArSIZ2yd94aD2T6dKvBkQLexROf++Xh4NdqFzSgBLWKbTF7PFNEX1rudVJUdFJPpcNw+t3ZvZUVLAbSIKbKOet19Kh6MBLQCx+Tq1aQz7o2mgoPlCI1y0CLCt71Bd/h2MlOH3Kg9mbwdjksRljEn33vXaWqUg1Yi3HuqGiWgu7Ht2rLGNqiuaWQB79+DvlcfQqsFug4fosBLGhLd1TlWebJzo9Z4liJI4iV7Bm7JfrV/LwmNHtLzea1xosfBmAXRHsiOPIJelB72A2zKQj1GhJukHZBgyqem4MsXFUMyymujiih7aYGgjGObtktkNNORZS/Ys4mMLBcqEeDpCTTg55MTgGttjW4wXGs+uodr7c+/4FqTfsq16GTKJJzw4MwV5z6Da81lczNO6l1rjC74HQqw6QfUFx4XZqZN12vqwbXmkGDtLQO41pZe6C/hWtuaXqEO8mGWSDua4ChTSJNUfuK41rQjObdmve5Qm9XWcflWzabrOvbMkNUlc9W/pswV1EOGg+yL2OkzozIrZspG2yWnRzb1PGxzk1Mz0+OAYW2XREPHNQfJKpvI57/dEnzXSl+5wtflmnbUK0KXpbvhOTROlE9qsj7ExKAbVFuzeoRCcqnT73XxHL4oX3yO2EpDn8CMp6ypRF2c5s7u8VJpVaFsusjEDhzq4Vr6GCGWLmx2pMe6liiBQm4EERc7pesbKQ8VmX1ALVAiY+e0pmmaDLjIsGJ9RdeRGAn50VQq5MYhARgJVH1OPM12Mm8U3J1wmxORcWtLSpcultKypPUlbdQaT2vPai7xwnsTrZ2fn9U4CmrLT6o7RwGYHSgMBOb9pwXs6qx6C5UVz2pp2yCVhrnRxLQ5GK+HlxbcooBIfz6pVRZu/5Imy9moNU6BBiDkNNCiyJDousUY7QgEXr7C2fLBZAQWdz4flvI1aRVzLbbKiuwljYEkAkMhahqqiwBpRu/Ub7kM/6Do1MViZyqgA8x8yginwSbS70lyIymOzm+286iQshyJLlg8pQ6Ns9uxSp8HyLNXccypCWuWJkjvaHDjUuSwJ4A+sTWSRbyVmmVJONgu9XAitEvCV+FcSmzBcQVzvu1la2IXoBtmYkl+dreYflaOWWBnccudryVpxhQuxbW3d25NlpEoIp+kWcgKdqfl4QJIj7aP0rKm5BU8j3+ZDnbRRvDKNNfo3pSFVT+fgMlkGccQ9PI4Rmc4OOudz4QNLh2Clm58LgE8Ovqp/lXXtEzAUfW8iorc9zAhw696KgS1Pxn1KnHKYvrvRMzXfswuH4ziJHOB09yYyo6FHwTTKoGqjiUMeppdjreSzBAuZCKfeIDvfWxz7CRJ3q14PAbdKgI1q2nZOpEy2c3voL8b1ZZjB0qmQhggT+U3pUjWyqifxtDfQ/0DKUW4UpqpmC0MyHLFE0IBMcGXDaAgPXi0I3n0CFNCKMv2xURdEZBmmCxMl8aBTLRqmvX7VDhrfenwi72UiwDU67nAQi3r19dkZT3y/dpXXSskB1p6rPbWyENLLA6wfGcjN7CuTa7Oznq/W5NZ7GcnqHZ0S3tdWO/etK/6U0Flq3aLQnfneJr4JxliptC0kgkVgDP3FJTKSTPVHdFFa19NX1uDaa+j1MClNX097LZsl2CPzxi2A8y1oveSGwIIaxY5UhobiEmWHQrvdK30PDo9ER7PdqL64WT3UthHYjj4ePNR7EHiySPER3ylHZWdiFuace7SeV0dstJDqAu/KvQI39QdwRPhb5nLkDi4tnaOsOcI/0L+BzPONO3i6pXVmfYTj1g6xP+uaarKrdV6Cd3MbaubcI5t7ia2mvG50P8r/Npq6dnjLPUZGKcBWuLIJEU+YXJKAmfgYY5Z7DHW4//ebuOXdpk0QOvSAK3HJJkUMul4ARW1RMSs1tSBf7uvAkqPlMNPpajUoX6zhXTPmDmrMt8YG5EPJXIZg0UWTuFi2ZRQiak1WBMW4BkJGAdZEQdkUeygamTkfRSh7muaFucpWx9zFk3jBL4/N/pRS5TQ+bg9ep2qvY9SVI14UIAv8OdfcFxbBshfWRHaY/gCPADTgWs9i6rd71vjd9U4kOviYLMHS7sz7b2xZt3e2OpMh+N3Sca2163EjEJxTnMSFYSh0CHYszE7fn/yoXyU0XA8bfe3EEb4hDwhN0voPQqW7GNRmUYAgGxObjE4JMC29BcCzFTBGXGEzgTjkOnVovDgVtpFhvPa3bOJnq+EPIJpQNagI2fBdFXpgj1HuPkuXQorKqaPLKKaK4GkNEqg7o2xnAH+qC2nNevG05rFE34Ck9Fg1un3ZGDT6oyt6WPIlXLGtZXV894rcoVFlVMqkx34omoVMByzumDFT0a9ngP4WFbMUVyylFm71ys996CiMkWVfQpKYI7sG2n8VI6GWLQ0iQdPvEyURhhf0d3N+CId8Q7nrfF5P9BXQWnJPM6Ih1wodkspV7d65dVQMdUD2RYX+JcFlSJJbHcPFsQtqfvnxO6BhEpY94XtP+sP31rjiGB9KyeraDp+3wx9HwfND8fit0vv5O+PDxdBgZy7Hl3mbY8vUz9I3WIWroA6/55Eeil6nwh5FD136UYabHH/WsZq+2fDxK/fvRr30ku+pyeNf5knT8XZudrMA+IUc4vxDo3dHFVOqu4Aq8nOsUvv4ttlNb06hLN1ZRtMT0hO3hcA04xFy4zPGlM5PcGWXQCmycLFgtybkdGY9AWj0gFKO93gjfRsTMdjmQ5Z3ydLjjoLzWQPpPQR57BzcBe2QzbRTmqkIVNBU94WKiZtMY9EVspINkh60HoyzEuXsixjhfP3x1XxuNS3TTDywnnQ2FHA0Yy3QBHJw+mIJH63xN5WRVwUv1PKwy25BQFpiksYR/LyUmW05ovKpZkmCpZQMPQyixy9OK5lcNsh43QtveqR4lBkAl6N+9Da6nycGyrj7+cqEOC4lkRMWrmW4zytW/UhJSUjeXqVixqT6nhsFOAFuRfDbPVMBtuBQd77P27lCl5KaZz0/rByRSi7sN6uJ+QTbmXBCzPPJ+6LZXF53NJNjVEjZ028K4YDWTnZKvYsDJOr1gDd+Jx78VXfMRBjq5ofzl1iX+ANe3/yoXaDN13EkRg0h6Y4tbLgDBjlr/cJY94mucR8RR056zJsxcmnlnqvC0b+MTcww8EtsfEoIJ5NfOQm0iEjSj05Yq579UjKJ8iBR68OGlEFrwrjKQSFMS9+mcRVD3srwfJj08BeYcYDxGkisdl3YxU2aRmZIUoGj0Ipewvcvm3wN5HLbmSGKAyer+Eq1nTtHy6NFHWkCV3zML+jwc3IDZfEaxXwHcO19rJa+cochHboFaM9eHbp+HwxQ5R1ebDjK/MRI+WZ67mCW/ihBfqJviP9npzmWyebPNKzH0nyqIPjHGD2FvT+Q09AJWIhtVdbYOzIj7lkCY5gi4BCS2drgEvAbhFx0Zy4hG9GsQlRVmqa6yqrbbu9yXQ8BB3dMDOchx4PzcbPtZNnegSak1PJiRGlsSJg8G8teP9ZF1peb4IuifEpdfUnoFPWJexGEH/+Sm9C4+TkifDAbOyqr0INlDwTb2l5aO5iR2/CArkMPwHdIYwHVG+CkSHyCejyIBSvi2z8ApyqT0E8AV0dbHkw8eYJ6GibUyncdtvXD8cRH/ZurkQg/rs2WUW4I71iti3kDvXwXkl/wHxW8kGy7A4uK/L7L+/k3Eh/114ur2+sqnvMb6U7RVCpwdSqwHC8j4S4lrNIQvJ+LwkjxNgdDZwiCTGG/4Hb4RsX9Y54WfVcIo0FQKWet4tJM2CHqufvULnRIlYp3dNfDle6lE03vkQa7RY9q3HzfKnQt1meVGvbcpb8f6Nrd+oioW3zSLuiY/J1ghLM8rtpCmdNf6iLYXsTH5WaOQ4jqAxGciNyTzBB4JGf0lPZE/lcHnc1kqTJN+50SZDsvdfBI45w7/hGXRJUQ/7PkdZvl6lvTnXtzVdFIa5HFS7lEyj1oSuyVlFGuUBbLCoHpy720PTNo38KAzxDzkOaTPkq2H1Je03r9K9UvMk6G7SKFWtxZr5zNZkOL6MvZybRg1Y+mBADl1e16LmyFr0UswoQtLZjBgn0nqFznXbgSa7StNJbNXFb4W5Mq+LOTDqk1R5cjWbDwcz6vTdtybMybUyY2xv0VP1gluExXH/YiasL0yBjws9RL0lk6KcnjV9ymYyEkxNr/KbXsWaX7UH73JLR6TSLeEjUXkuunccXrCWmcQZNIS+RUFidvtwf7E8Kj1XxQVnfqCmBzFc8VIb/Y8YUEiNx2VZ16iTueWG9myUpku1u2QTKjlV4c5ntrifVYl5mG2zxPN/JKKs7S4acvJYf21VVrcSZBQwliFW8u4xZqkU7kgW0bOPZsuoRbjx658HCRTcYEAc/wEwYIXd1aPxSO3lm9qcTMIGt5Hec1QcUMXL5agMJolVAPfJJfRE4XrLe4OJ8OJtc9EYtPeo8xyAr3zkVB7sT2hgiyxP6FDmvkIs8Gwfsy6GD5QD5CoNNgwDbHKKsGdiu+hSRSlViR9e0JQXs3WrqQs+cUs54gHz1GJ/GZvZiUAQpTsboBDQXoNdkJVOqV3cefuprV5nCavkx8U3lPY12ETItta1BB3nHHILQA+vUgilmfKvW9J+5PG9l7hwkV/eTmvitG9GVub8mGDuy6wUsqWoU3dKnLcCyM6lZEX8vdi6o+CYYhTfbHaJzSYJGvyuAoqh608g9bwMXDxIwtt9t9So9tgVRpQ07uyu6in3LqS2q8iYYxVdbXXKqW3TIvdi6XV6imZtglL3ee3l+3x35+HBughH/3Lopndeyza2E484r0d986/mQC86Vpzhk+xcb92LJnUtlmHIAW/emq22UJhg7Wg9AVKRsH0jZjfD4GG2q1GD8WHX9u3AgN9Pb3oWWLQTFq9rGnhvZmWvi1ffAH+5jDn/fmaGOT44ZN9P8Ffz4MjqTZ6JFeL6hy5mmja3JVX/aMn7TNHsl7BJzDMmdNCjeYFMFxuYYwlZwd7/VrB3Bq6iIx3OUH03k5QhFkvzfVQga5AnGQtvGjAG+xR6QRVTWxLj84h+TX9R9oh3FLZmOdAGZiS3kR4iI/HQeZyq6IzrEVR42dYqXosZy8mprXfWneuRmqycwlxye5o2L6KRPjAf5fCJO/v8bAAD//5kAvuhKZQAA'))]", "computerName": "[variables('vmName')]", "adminUsername": "[parameters('linuxAdminUsername')]", "linuxConfiguration": { @@ -298,10 +444,10 @@ }, "storageProfile": { "imageReference": { - "publisher": "[variables('imagePublisher')]", - "offer": "[variables('imageOffer')]", - "sku": "[variables('imageSku')]", - "version": "[variables('imageVersion')]" + "publisher": "[parameters('dvmImagePublisher')]", + "offer": "[parameters('dvmImageOffer')]", + "sku": "[parameters('dvmImageSku')]", + "version": "[parameters('dvmImageVersion')]" }, "osDisk": { "name": "osdisk", diff --git a/kubernetes/template/DeploymentTemplates/azuredeploy.parameters.json b/kubernetes/template/DeploymentTemplates/azuredeploy.parameters.json index adbf847..b42d6a5 100644 --- a/kubernetes/template/DeploymentTemplates/azuredeploy.parameters.json +++ b/kubernetes/template/DeploymentTemplates/azuredeploy.parameters.json @@ -35,12 +35,27 @@ "kubernetesAzureCloudProviderVersion": { "value": "" }, + "kubernetesAzureCloudProviderRelease": { + "value": "" + }, "identitySystem": { "value": "" }, "nodeDistro": { + "value": "ubuntu" + }, + "customVnetName": { + "value": "" + }, + "masterSubnetName": { + "value": "" + }, + "agentSubnetName": { "value": "" - }, + }, + "firstConsecutiveStaticIP": { + "value": "" + }, "aksEngineBaseURL": { "value": "" }, @@ -56,8 +71,38 @@ "clusterDefinitionFileName": { "value": "" }, - "kubernetesImageBase": { + "networkPlugin": { + "value": "kubenet" + }, + "networkPolicy": { + "value": "" + }, + "availabilityProfile": { + "value": "AvailabilitySet" + }, + "windowsAgentPoolProfileCount": { + "value": "0" + }, + "windowsAgentPoolProfileVMSize": { + "value": "Standard_D2_v2" + }, + "windowsAdminUsername": { + "value": "azureuser" + }, + "windowsAdminPassword": { + "value": "" + }, + "customWindowsPackageURL": { + "value": "" + }, + "enableTillerAddOn": { + "value": "false" + }, + "containerRuntime": { + "value": "docker" + }, + "localAKSeBinaryURL": { "value": "" } } -} \ No newline at end of file +} diff --git a/kubernetes/template/DeploymentTemplates/clusterDefinition.json b/kubernetes/template/DeploymentTemplates/clusterDefinition.json index 583c234..68c8db8 100644 --- a/kubernetes/template/DeploymentTemplates/clusterDefinition.json +++ b/kubernetes/template/DeploymentTemplates/clusterDefinition.json @@ -8,7 +8,31 @@ "kubernetesConfig": { "kubernetesImageBase": "", "useInstanceMetadata": false, - "networkPlugin": "flannel" + "networkPlugin": "kubenet", + "networkPolicy": "", + "containerRuntime": "docker", + "cloudProviderBackoff": true, + "cloudProviderBackoffRetries": 1, + "cloudProviderBackoffDuration": 30, + "cloudProviderRateLimit": true, + "cloudProviderRateLimitQPS": 3, + "cloudProviderRateLimitBucket": 10, + "cloudProviderRateLimitQPSWrite": 3, + "cloudProviderRateLimitBucketWrite": 10, + "kubeletConfig": { + "--node-status-update-frequency": "1m" + }, + "controllerManagerConfig": { + "--node-monitor-grace-period": "5m", + "--pod-eviction-timeout": "5m", + "--route-reconciliation-period": "1m" + }, + "addons": [ + { + "name": "tiller", + "enabled": false + } + ] } }, "customCloudProfile": { @@ -18,20 +42,11 @@ "dnsPrefix": "", "distro": "ubuntu", "osDiskSizeGB": 200, + "availabilityProfile": "AvailabilitySet", "count": 3, "vmSize": "Standard_D2_v2" }, - "agentPoolProfiles": [ - { - "name": "linuxpool", - "osDiskSizeGB": 200, - "count": 3, - "vmSize": "Standard_D2_v2", - "distro": "ubuntu", - "availabilityProfile": "AvailabilitySet", - "AcceleratedNetworkingEnabled": false - } - ], + "agentPoolProfiles": [], "linuxProfile": { "adminUsername": "azureuser", "ssh": { @@ -42,9 +57,14 @@ ] } }, + "windowsProfile": { + "adminUsername": "azureuser", + "adminPassword": "", + "sshEnabled": true + }, "servicePrincipalProfile": { "clientId": "", "secret": "" } } -} +} \ No newline at end of file diff --git a/kubernetes/template/DeploymentTemplates/createUiDefinition.json b/kubernetes/template/DeploymentTemplates/createUiDefinition.json index e859ac9..da7818a 100644 --- a/kubernetes/template/DeploymentTemplates/createUiDefinition.json +++ b/kubernetes/template/DeploymentTemplates/createUiDefinition.json @@ -2,492 +2,757 @@ "handler": "Microsoft.Compute.MultiVm", "version": "0.4.0-preview", "parameters": { - "basics": [ - ], - "steps": [{ - "name": "KubernetesClusterSettings", - "label": "Kubernetes Cluster settings", - "subLabel": { - "preValidation": "Provide Kubernetes Cluster settings.", - "postValidation": "Done" - }, - "bladeTitle": "Kubernetes Cluster Settings", - "elements": [{ - "name": "linuxAdminUsername", - "type": "Microsoft.Common.TextBox", - "label": "Linux VM admin username", - "defaultValue": "azureuser", - "toolTip": "User name for the Linux Virtual Machines that are part of the Kubernetes cluster and DVM.", - "constraints": { - "required": true - } - }, + "basics": [], + "steps": [ { - "name": "sshPublicKey", - "type": "Microsoft.Compute.CredentialsCombo", - "label": { - "authenticationType": "Authentication type", - "password": "Password", - "confirmPassword": "Confirm password", - "sshPublicKey": "SSH public key" - }, - "toolTip": { - "authenticationType": "Authentication type", - "password": "Password", - "sshPublicKey": "SSH public key used to connect to virtual Linux machines created as part of the Kubernetes cluster and DVM." - }, - "constraints": { - "required": true - }, - "options": { - "hideConfirmation": false, - "hidePassword": true + "name": "KubernetesClusterSettings", + "label": "Kubernetes Cluster settings", + "subLabel": { + "preValidation": "Provide Kubernetes Cluster settings.", + "postValidation": "Done" }, - "osPlatform": "Linux", - "visible": true - }, - { - "name": "masterProfileDnsPrefix", - "type": "Microsoft.Common.TextBox", - "label": "Master Profile DNS prefix", - "defaultValue": null, - "toolTip": "This must be a region-unique name, for example, k8s-12345. A best practice is to use the same prefix for the DNS and region name.", - "constraints": { - "required": true, - "regex": "^[a-z][a-z0-9-]{1,61}[a-z0-9]$", - "validationMessage": "DNS prefix is invalid. The prefix must conform to the following regular expression: ^[a-z][a-z0-9-]{1,61}[a-z0-9]$." - } - }, - { - "name": "masterPoolProfileCount", - "type": "Microsoft.Common.DropDown", - "label": "Kubernetes master pool profile count", - "defaultValue": "3", - "toolTip": "The number of master nodes for the Kubernetes cluster. This value should be odd number.", - "constraints": { - "allowedValues": [{ - "label": "1", - "value": "1" - }, - { - "label": "3", - "value": "3" - }, - { - "label": "5", - "value": "5" - }, - { - "label": "7", - "value": "7" + "bladeTitle": "Kubernetes Cluster Settings", + "elements": [ + { + "name": "linuxAdminUsername", + "type": "Microsoft.Common.TextBox", + "label": "Linux VM admin username", + "defaultValue": "azureuser", + "toolTip": "User name for the Linux virtual machines that are part of the Kubernetes cluster and DVM.", + "constraints": { + "required": true } - ] - }, - "visible": true - }, - { - "name": "masterPoolProfileVMSize", - "type": "Microsoft.Common.DropDown", - "label": "The VMSize of Kubernetes master VMs", - "defaultValue": "Standard_D2_v2", - "toolTip": "The VMSize of Kubernetes master VMs", - "constraints": { - "allowedValues": [ - { - "label": "Standard_D1_v2", - "value": "Standard_D1_v2" - }, - { - "label": "Standard_D2_v2", - "value": "Standard_D2_v2" - }, - { - "label": "Standard_D3_v2", - "value": "Standard_D3_v2" - }, - { - "label": "Standard_D4_v2", - "value": "Standard_D4_v2" - }, - { - "label": "Standard_D5_v2", - "value": "Standard_D5_v2" - }, - { - "label": "Standard_D11_v2", - "value": "Standard_D11_v2" - }, - { - "label": "Standard_D12_v2", - "value": "Standard_D12_v2" - }, - { - "label": "Standard_D13_v2", - "value": "Standard_D13_v2" - }, - { - "label": "Standard_D14_v2", - "value": "Standard_D14_v2" - }, - { - "label": "Standard_DS1_v2", - "value": "Standard_DS1_v2" - }, - { - "label": "Standard_DS2_v2", - "value": "Standard_DS2_v2" - }, - { - "label": "Standard_DS3_v2", - "value": "Standard_DS3_v2" - }, - { - "label": "Standard_DS4_v2", - "value": "Standard_DS4_v2" - }, - { - "label": "Standard_DS5_v2", - "value": "Standard_DS5_v2" - }, - { - "label": "Standard_DS11_v2", - "value": "Standard_DS11_v2" - }, - { - "label": "Standard_DS12_v2", - "value": "Standard_DS12_v2" - }, - { - "label": "Standard_DS13_v2", - "value": "Standard_DS13_v2" - }, - { - "label": "Standard_DS14_v2", - "value": "Standard_DS14_v2" + }, + { + "name": "sshPublicKey", + "type": "Microsoft.Compute.CredentialsCombo", + "label": { + "authenticationType": "Authentication type", + "password": "Password", + "confirmPassword": "Confirm password", + "sshPublicKey": "SSH public key" + }, + "toolTip": { + "authenticationType": "Authentication type", + "password": "Password", + "sshPublicKey": "SSH public key used to connect to virtual Linux machines created as part of the Kubernetes cluster and DVM." + }, + "constraints": { + "required": true + }, + "options": { + "hideConfirmation": false, + "hidePassword": true + }, + "osPlatform": "Linux", + "visible": true + }, + { + "name": "masterProfileDnsPrefix", + "type": "Microsoft.Common.TextBox", + "label": "Master Profile DNS prefix", + "defaultValue": null, + "toolTip": "This must be a region-unique name, for example, k8s-12345. A best practice is to use the same prefix for the DNS and region name.", + "constraints": { + "required": true, + "regex": "^[a-z][a-z0-9-]{1,61}[a-z0-9]$", + "validationMessage": "DNS prefix is invalid. The prefix must conform to the following regular expression: ^[a-z][a-z0-9-]{1,61}[a-z0-9]$." } - ] - }, - "visible": true - }, - { - "name": "agentPoolProfileCount", - "type": "Microsoft.Common.DropDown", - "label": "Kubernetes node pool profile count", - "defaultValue": "3", - "toolTip": "The number of agents for the Kubernetes cluster.", - "constraints": { - "allowedValues": [{ - "label": "1", - "value": "1" - }, - { - "label": "2", - "value": "2" - }, - { - "label": "3", - "value": "3" - }, - { - "label": "4", - "value": "4" - }, - { - "label": "5", - "value": "5" - }, - { - "label": "6", - "value": "6" - }, - { - "label": "7", - "value": "7" - }, { - "label": "8", - "value": "8" - }, - { - "label": "9", - "value": "9" - }, - { - "label": "10", - "value": "10" - }, - { - "label": "11", - "value": "11" - }, { - "label": "12", - "value": "12" - }, - { - "label": "13", - "value": "13" - }, - { - "label": "14", - "value": "14" - }, - { - "label": "15", - "value": "15" - }, - { - "label": "16", - "value": "16" + }, + { + "name": "masterPoolProfileCount", + "type": "Microsoft.Common.DropDown", + "label": "Kubernetes master pool profile count", + "defaultValue": "3", + "toolTip": "The number of master nodes for the Kubernetes cluster. This value should be odd number.", + "constraints": { + "allowedValues": [ + { + "label": "1", + "value": "1" + }, + { + "label": "3", + "value": "3" + }, + { + "label": "5", + "value": "5" + }, + { + "label": "7", + "value": "7" + } + ] + }, + "visible": true + }, + { + "name": "masterPoolProfileVMSize", + "type": "Microsoft.Common.DropDown", + "label": "The virtual machine size of the Kubernetes master nodes", + "defaultValue": "Standard_D2_v2", + "toolTip": "The virtual machine size of the Kubernetes master nodes", + "constraints": { + "allowedValues": [ + { + "label": "Standard_D1_v2", + "value": "Standard_D1_v2" + }, + { + "label": "Standard_D2_v2", + "value": "Standard_D2_v2" + }, + { + "label": "Standard_D3_v2", + "value": "Standard_D3_v2" + }, + { + "label": "Standard_D4_v2", + "value": "Standard_D4_v2" + }, + { + "label": "Standard_D5_v2", + "value": "Standard_D5_v2" + }, + { + "label": "Standard_D11_v2", + "value": "Standard_D11_v2" + }, + { + "label": "Standard_D12_v2", + "value": "Standard_D12_v2" + }, + { + "label": "Standard_D13_v2", + "value": "Standard_D13_v2" + }, + { + "label": "Standard_D14_v2", + "value": "Standard_D14_v2" + }, + { + "label": "Standard_DS1_v2", + "value": "Standard_DS1_v2" + }, + { + "label": "Standard_DS2_v2", + "value": "Standard_DS2_v2" + }, + { + "label": "Standard_DS3_v2", + "value": "Standard_DS3_v2" + }, + { + "label": "Standard_DS4_v2", + "value": "Standard_DS4_v2" + }, + { + "label": "Standard_DS5_v2", + "value": "Standard_DS5_v2" + }, + { + "label": "Standard_DS11_v2", + "value": "Standard_DS11_v2" + }, + { + "label": "Standard_DS12_v2", + "value": "Standard_DS12_v2" + }, + { + "label": "Standard_DS13_v2", + "value": "Standard_DS13_v2" + }, + { + "label": "Standard_DS14_v2", + "value": "Standard_DS14_v2" + } + ] + }, + "visible": true + }, + { + "name": "agentPoolProfileCount", + "type": "Microsoft.Common.DropDown", + "label": "Kubernetes linux node pool profile count", + "defaultValue": "3", + "toolTip": "The number of linux agents for Kubernetes cluster.", + "constraints": { + "allowedValues": [ + { + "label": "1", + "value": "1" + }, + { + "label": "2", + "value": "2" + }, + { + "label": "3", + "value": "3" + }, + { + "label": "4", + "value": "4" + }, + { + "label": "5", + "value": "5" + }, + { + "label": "6", + "value": "6" + }, + { + "label": "7", + "value": "7" + }, + { + "label": "8", + "value": "8" + }, + { + "label": "9", + "value": "9" + }, + { + "label": "10", + "value": "10" + }, + { + "label": "11", + "value": "11" + }, + { + "label": "12", + "value": "12" + }, + { + "label": "13", + "value": "13" + }, + { + "label": "14", + "value": "14" + }, + { + "label": "15", + "value": "15" + }, + { + "label": "16", + "value": "16" + } + ] + }, + "visible": true + }, + { + "name": "agentPoolProfileVMSize", + "type": "Microsoft.Common.DropDown", + "label": "The virtual machine size of Kubernetes linux agent nodes", + "defaultValue": "Standard_D2_v2", + "toolTip": "The virtual machine size of Kubernetes linux agent nodes", + "constraints": { + "allowedValues": [ + { + "label": "Standard_D1_v2", + "value": "Standard_D1_v2" + }, + { + "label": "Standard_D2_v2", + "value": "Standard_D2_v2" + }, + { + "label": "Standard_D3_v2", + "value": "Standard_D3_v2" + }, + { + "label": "Standard_D4_v2", + "value": "Standard_D4_v2" + }, + { + "label": "Standard_D5_v2", + "value": "Standard_D5_v2" + }, + { + "label": "Standard_D11_v2", + "value": "Standard_D11_v2" + }, + { + "label": "Standard_D12_v2", + "value": "Standard_D12_v2" + }, + { + "label": "Standard_D13_v2", + "value": "Standard_D13_v2" + }, + { + "label": "Standard_D14_v2", + "value": "Standard_D14_v2" + }, + { + "label": "Standard_DS1_v2", + "value": "Standard_DS1_v2" + }, + { + "label": "Standard_DS2_v2", + "value": "Standard_DS2_v2" + }, + { + "label": "Standard_DS3_v2", + "value": "Standard_DS3_v2" + }, + { + "label": "Standard_DS4_v2", + "value": "Standard_DS4_v2" + }, + { + "label": "Standard_DS5_v2", + "value": "Standard_DS5_v2" + }, + { + "label": "Standard_DS11_v2", + "value": "Standard_DS11_v2" + }, + { + "label": "Standard_DS12_v2", + "value": "Standard_DS12_v2" + }, + { + "label": "Standard_DS13_v2", + "value": "Standard_DS13_v2" + }, + { + "label": "Standard_DS14_v2", + "value": "Standard_DS14_v2" + } + ] + }, + "visible": true + }, + { + "name": "storageProfile", + "type": "Microsoft.Common.DropDown", + "label": "The Storage Profile", + "defaultValue": "Managed disk", + "toolTip": "The storage profile of Kubernetes node VMs.", + "constraints": { + "required": true, + "allowedValues": [ + { + "label": "Managed disk", + "value": "manageddisk" + } + ] + }, + "visible": false + }, + { + "name": "identitySystem", + "type": "Microsoft.Common.DropDown", + "label": "Azure Stack identity system", + "defaultValue": "AzureAD", + "toolTip": "The identity sytem of Azure Stack.", + "constraints": { + "allowedValues": [ + { + "label": "AzureAD", + "value": "AzureAD" + }, + { + "label": "ADFS", + "value": "ADFS" + } + ] + }, + "visible": true + }, + { + "name": "nodeDistro", + "type": "Microsoft.Common.DropDown", + "label": "Kubernetes node distro", + "defaultValue": "Ubuntu", + "toolTip": "Kubernetes node distro.", + "constraints": { + "allowedValues": [ + { + "label": "Ubuntu", + "value": "ubuntu" + }, + { + "label": "AKS", + "value": "aks-ubuntu-16.04" + } + ] + }, + "visible": false + }, + { + "name": "servicePrincipalClientId", + "type": "Microsoft.Common.TextBox", + "label": "Service principal clientId", + "defaultValue": null, + "toolTip": "The service principal application ID (used by the Kubernetes Azure cloud provider). More help here: https://github.com/Azure/aks-engine/blob/master/docs/topics/service-principals.md", + "constraints": { + "required": true } - ] - }, - "visible": true - }, - { - "name": "agentPoolProfileVMSize", - "type": "Microsoft.Common.DropDown", - "label": "The VMSize of Kubernetes node VMs", - "defaultValue": "Standard_D2_v2", - "toolTip": "The VMSize of Kubernetes node VMs", - "constraints": { - "allowedValues": [ - { - "label": "Standard_D1_v2", - "value": "Standard_D1_v2" - }, - { - "label": "Standard_D2_v2", - "value": "Standard_D2_v2" - }, - { - "label": "Standard_D3_v2", - "value": "Standard_D3_v2" - }, - { - "label": "Standard_D4_v2", - "value": "Standard_D4_v2" - }, - { - "label": "Standard_D5_v2", - "value": "Standard_D5_v2" - }, - { - "label": "Standard_D11_v2", - "value": "Standard_D11_v2" - }, - { - "label": "Standard_D12_v2", - "value": "Standard_D12_v2" - }, - { - "label": "Standard_D13_v2", - "value": "Standard_D13_v2" - }, - { - "label": "Standard_D14_v2", - "value": "Standard_D14_v2" - }, - { - "label": "Standard_DS1_v2", - "value": "Standard_DS1_v2" - }, - { - "label": "Standard_DS2_v2", - "value": "Standard_DS2_v2" - }, - { - "label": "Standard_DS3_v2", - "value": "Standard_DS3_v2" - }, - { - "label": "Standard_DS4_v2", - "value": "Standard_DS4_v2" - }, - { - "label": "Standard_DS5_v2", - "value": "Standard_DS5_v2" - }, - { - "label": "Standard_DS11_v2", - "value": "Standard_DS11_v2" - }, - { - "label": "Standard_DS12_v2", - "value": "Standard_DS12_v2" - }, - { - "label": "Standard_DS13_v2", - "value": "Standard_DS13_v2" - }, - { - "label": "Standard_DS14_v2", - "value": "Standard_DS14_v2" - } - ] - }, - "visible": true - }, - { - "name": "storageProfile", - "type": "Microsoft.Common.DropDown", - "label": "The Storage Profile", - "defaultValue": "Managed disk", - "toolTip": "The storage profile of Kubernetes node VMs.", - "constraints": { - "required": true, - "allowedValues": [ - { - "label": "Managed disk", - "value": "manageddisk" + }, + { + "name": "servicePrincipalClientSecret", + "type": "Microsoft.Common.TextBox", + "label": "Service principal client secret", + "defaultValue": null, + "toolTip": "The service principal client secret.", + "constraints": { + "required": true } - ] - }, - "visible": false - }, - { - "name": "identitySystem", - "type": "Microsoft.Common.DropDown", - "label": "Azure Stack identity system", - "defaultValue": "AzureAD", - "toolTip": "The identity sytem of Azure Stack.", - "constraints": { - "allowedValues": [{ - "label": "AzureAD", - "value": "AzureAD" - }, - { - "label": "ADFS", - "value": "ADFS" - } - ] - }, - "visible": true - }, - { - "name": "nodeDistro", - "type": "Microsoft.Common.DropDown", - "label": "Kubernetes node distro", - "defaultValue": "Ubuntu", - "toolTip": "Kubernetes node distro.", - "constraints": { - "allowedValues": [{ - "label": "Ubuntu", - "value": "ubuntu" - }, - { - "label": "AKS", - "value": "aks" - } - ] - }, - "visible": false - }, - { - "name": "servicePrincipalClientId", - "type": "Microsoft.Common.TextBox", - "label": "Service principal clientId", - "defaultValue": null, - "toolTip": "The service principal application ID (used by the Kubernetes Azure cloud provider). More help here: https://github.com/Azure/aks-engine/blob/master/docs/topics/service-principals.md", - "constraints": { - "required": true - } - }, - { - "name": "servicePrincipalClientSecret", - "type": "Microsoft.Common.TextBox", - "label": "Service principal client secret", - "defaultValue": null, - "toolTip": "The service principal client secret.", - "constraints": { - "required": true - } - }, - { - "name": "kubernetesAzureCloudProviderVersion", - "type": "Microsoft.Common.DropDown", - "label": "Kubernetes version", - "defaultValue": "1.12", - "toolTip": "This is the version for the Kubernetes Azure cloud provider.", - "constraints": { - "allowedValues": [ - { - "label": "1.12", - "value": "1.12" - }, - { - "label": "1.13", - "value": "1.13" - } - ] - }, - "visible": true - }, - { - "name": "aksEngineBaseURL", - "type": "Microsoft.Common.TextBox", - "label": "AKS Engine URL base", - "defaultValue": "https://github.com/Azure/aks-engine/releases/download", - "toolTip": "The beginning of the URL for downloading the AKS Engine binary", - "constraints": { - "required": true - }, - "visible": false - }, - { - "name": "aksEngineReleaseVersion", - "type": "Microsoft.Common.TextBox", - "label": "AKS Engine release version", - "defaultValue": "v0.37.4", - "toolTip": "The version of AKS Engine to download", - "constraints": { - "required": true - }, - "visible": false - }, - { - "name": "galleryRepository", - "type": "Microsoft.Common.TextBox", - "label": "Gallery repository", - "defaultValue": "msazurestackworkloads/azurestack-gallery", - "toolTip": "The name of the repository for the Marketplace Item", - "constraints": { - "required": true - }, - "visible": false - }, - { - "name": "galleryBranch", - "type": "Microsoft.Common.TextBox", - "label": "Gallery branch", - "defaultValue": "master", - "toolTip": "The name of the branch of the Gallery repository", - "constraints": { - "required": true - }, - "visible": false - }, - { - "name": "clusterDefinitionFileName", - "type": "Microsoft.Common.TextBox", - "label": "Cluster definition file name", - "defaultValue": "clusterDefinition.json", - "toolTip": "The name of the file containing the cluster definition", - "constraints": { - "required": true - }, - "visible": false - }, - { - "name": "kubernetesImageBase", - "type": "Microsoft.Common.TextBox", - "label": "Kubernetes Image Base", - "defaultValue": "mcr.microsoft.com/k8s/azurestack/core/", - "toolTip": "The base for Kubernetes images", - "constraints": { - "required": true - }, - "visible": false - }] - }], + }, + { + "name": "kubernetesAzureCloudProviderRelease", + "type": "Microsoft.Common.DropDown", + "label": "Kubernetes Release", + "defaultValue": "1.14", + "toolTip": "This is the release for the Kubernetes Azure cloud provider.", + "constraints": { + "allowedValues": [ + { + "label": "1.14", + "value": "1.14" + }, + { + "label": "1.15", + "value": "1.15" + } + ] + }, + "visible": true + }, + { + "name": "aksEngineBaseURL", + "type": "Microsoft.Common.TextBox", + "label": "AKS Engine URL base", + "defaultValue": "https://github.com/Azure/aks-engine/releases/download", + "toolTip": "The beginning of the URL for downloading the AKS Engine binary", + "constraints": { + "required": true + }, + "visible": false + }, + { + "name": "aksEngineReleaseVersion", + "type": "Microsoft.Common.TextBox", + "label": "AKS Engine release version", + "defaultValue": "v0.48.0", + "toolTip": "The version of AKS Engine to download", + "constraints": { + "required": true + }, + "visible": false + }, + { + "name": "galleryRepository", + "type": "Microsoft.Common.TextBox", + "label": "Gallery repository", + "defaultValue": "msazurestackworkloads/azurestack-gallery", + "toolTip": "The name of the repository for the Marketplace Item", + "constraints": { + "required": true + }, + "visible": false + }, + { + "name": "galleryBranch", + "type": "Microsoft.Common.TextBox", + "label": "Gallery branch", + "defaultValue": "master", + "toolTip": "The name of the branch of the Gallery repository", + "constraints": { + "required": true + }, + "visible": false + }, + { + "name": "clusterDefinitionFileName", + "type": "Microsoft.Common.TextBox", + "label": "Cluster definition file name", + "defaultValue": "clusterDefinition.json", + "toolTip": "The name of the file containing the cluster definition", + "constraints": { + "required": true + }, + "visible": false + }, + { + "name": "enableTillerAddOn", + "type": "Microsoft.Common.TextBox", + "label": "Flag to enable Tiller addon", + "defaultValue": "false", + "toolTip": "Flag to enable Tiller addon", + "visible": false + }, + { + "name": "containerRuntime", + "type": "Microsoft.Common.DropDown", + "label": "Container runtime", + "toolTip": "Container runtime to deploy on each cluster node", + "defaultValue": "docker", + "constraints": { + "allowedValues": [ + { + "label": "docker", + "value": "docker" + }, + { + "label": "containerd", + "value": "containerd" + } + ] + }, + "visible": false + }, + { + "name": "networkPlugin", + "type": "Microsoft.Common.DropDown", + "label": "Network Plugin name", + "defaultValue": "Kubenet", + "toolTip": "Network plugin which will deployed in Kubernetes cluster", + "constraints": { + "allowedValues": [ + { + "label": "Flannel", + "value": "flannel" + }, + { + "label": "Azure CNI", + "value": "azure" + }, + { + "label": "Kubenet", + "value": "kubenet" + } + ] + }, + "visible": false + }, + { + "name": "networkPolicy", + "type": "Microsoft.Common.DropDown", + "label": "Network Policy name", + "defaultValue": "", + "toolTip": "Network policy which will deployed in Kubernetes cluster", + "constraints": { + "allowedValues": [ + { + "label": "default", + "value": "" + }, + { + "label": "Azure CNI", + "value": "azure" + } + ] + }, + "visible": false + }, + { + "name": "availabilityProfile", + "type": "Microsoft.Common.DropDown", + "label": "Availability profile name", + "defaultValue": "Availability Set", + "toolTip": "Availability profile that nodes in the Kubernetes cluster will be deployed with", + "constraints": { + "allowedValues": [ + { + "label": "Availability Set", + "value": "AvailabilitySet" + }, + { + "label": "Virtual Machine Scale Sets", + "value": "VirtualMachineScaleSets" + } + ] + }, + "visible": false + }, + { + "name": "windowsAgentPoolProfileCount", + "type": "Microsoft.Common.TextBox", + "label": "Kubernetes Windows node pool profile count", + "defaultValue": "0", + "toolTip": "The number of Windows agent nodes in Kubernetes cluster.", + "constraints": { + "allowedValues": [ + { + "label": "0", + "value": "0" + }, + { + "label": "1", + "value": "1" + }, + { + "label": "2", + "value": "2" + }, + { + "label": "3", + "value": "3" + }, + { + "label": "4", + "value": "4" + }, + { + "label": "5", + "value": "5" + }, + { + "label": "6", + "value": "6" + }, + { + "label": "7", + "value": "7" + }, + { + "label": "8", + "value": "8" + }, + { + "label": "9", + "value": "9" + }, + { + "label": "10", + "value": "10" + } + ] + }, + "visible": false + }, + { + "name": "windowsAgentPoolProfileVMSize", + "type": "Microsoft.Common.DropDown", + "label": "The virtual machine size of the Kubernetes Windows agent nodes", + "defaultValue": "Standard_D2_v2", + "toolTip": "The virtual machine size of the Kubernetes Windows agent nodes", + "constraints": { + "allowedValues": [ + { + "label": "Standard_D1_v2", + "value": "Standard_D1_v2" + }, + { + "label": "Standard_D2_v2", + "value": "Standard_D2_v2" + }, + { + "label": "Standard_D3_v2", + "value": "Standard_D3_v2" + }, + { + "label": "Standard_D4_v2", + "value": "Standard_D4_v2" + }, + { + "label": "Standard_D5_v2", + "value": "Standard_D5_v2" + }, + { + "label": "Standard_D11_v2", + "value": "Standard_D11_v2" + }, + { + "label": "Standard_D12_v2", + "value": "Standard_D12_v2" + }, + { + "label": "Standard_D13_v2", + "value": "Standard_D13_v2" + }, + { + "label": "Standard_D14_v2", + "value": "Standard_D14_v2" + }, + { + "label": "Standard_DS1_v2", + "value": "Standard_DS1_v2" + }, + { + "label": "Standard_DS2_v2", + "value": "Standard_DS2_v2" + }, + { + "label": "Standard_DS3_v2", + "value": "Standard_DS3_v2" + }, + { + "label": "Standard_DS4_v2", + "value": "Standard_DS4_v2" + }, + { + "label": "Standard_DS5_v2", + "value": "Standard_DS5_v2" + }, + { + "label": "Standard_DS11_v2", + "value": "Standard_DS11_v2" + }, + { + "label": "Standard_DS12_v2", + "value": "Standard_DS12_v2" + }, + { + "label": "Standard_DS13_v2", + "value": "Standard_DS13_v2" + }, + { + "label": "Standard_DS14_v2", + "value": "Standard_DS14_v2" + } + ] + }, + "visible": false + }, + { + "name": "windowsAdminUsername", + "type": "Microsoft.Common.TextBox", + "label": "Windows VM admin username", + "defaultValue": "azureuser", + "toolTip": "User name for Windows virtual machines that are part of Kubernetes cluster.", + "visible": false + }, + { + "name": "windowsAdminPassword", + "type": "Microsoft.Common.TextBox", + "label": "Windows VM admin password", + "defaultValue": "", + "toolTip": "Password for Windows virtual machines that are part of Kubernetes cluster.", + "visible": false + }, + { + "name": "customWindowsPackageURL", + "type": "Microsoft.Common.TextBox", + "label": "Custom Windows package URL", + "defaultValue": "", + "toolTip": "Custom Windows K8s zip location which will be used to deploy(kubelet, kubeproxy) on Windows node.", + "visible": false + }, + { + "name": "localAKSeBinaryURL", + "type": "Microsoft.Common.TextBox", + "label": "AKS Engine Base URL (Disconnected)", + "defaultValue": "", + "toolTip": "The local blob URL AKS Engine. See list of supported AKS Engine on Azure Stack here. https://github.com/Azure/aks-engine/blob/master/docs/topics/azure-stack.md", + "constraints": { + "required": false + }, + "visible": true + } + ] + } + ], "outputs": { "linuxAdminUsername": "[steps('KubernetesClusterSettings').linuxAdminUsername]", "sshPublicKey": "[steps('KubernetesClusterSettings').sshPublicKey.sshPublicKey]", @@ -501,13 +766,23 @@ "identitySystem": "[steps('KubernetesClusterSettings').identitySystem]", "nodeDistro": "[steps('KubernetesClusterSettings').nodeDistro]", "servicePrincipalClientSecret": "[steps('KubernetesClusterSettings').servicePrincipalClientSecret]", - "kubernetesAzureCloudProviderVersion": "[steps('KubernetesClusterSettings').kubernetesAzureCloudProviderVersion]", + "kubernetesAzureCloudProviderRelease": "[steps('KubernetesClusterSettings').kubernetesAzureCloudProviderRelease]", "aksEngineBaseURL": "[steps('KubernetesClusterSettings').aksEngineBaseURL]", "aksEngineReleaseVersion": "[steps('KubernetesClusterSettings').aksEngineReleaseVersion]", "galleryRepository": "[steps('KubernetesClusterSettings').galleryRepository]", "galleryBranch": "[steps('KubernetesClusterSettings').galleryBranch]", "clusterDefinitionFileName": "[steps('KubernetesClusterSettings').clusterDefinitionFileName]", - "kubernetesImageBase": "[steps('KubernetesClusterSettings').kubernetesImageBase]" + "networkPlugin": "[steps('KubernetesClusterSettings').networkPlugin]", + "networkPolicy": "[steps('KubernetesClusterSettings').networkPolicy]", + "enableTillerAddOn": "[steps('KubernetesClusterSettings').enableTillerAddOn]", + "containerRuntime": "[steps('KubernetesClusterSettings').containerRuntime]", + "availabilityProfile": "[steps('KubernetesClusterSettings').availabilityProfile]", + "windowsAgentPoolProfileCount": "[steps('KubernetesClusterSettings').windowsAgentPoolProfileCount]", + "windowsAgentPoolProfileVMSize": "[steps('KubernetesClusterSettings').windowsAgentPoolProfileVMSize]", + "windowsAdminUsername": "[steps('KubernetesClusterSettings').windowsAdminUsername]", + "windowsAdminPassword": "[steps('KubernetesClusterSettings').windowsAdminPassword]", + "customWindowsPackageURL": "[steps('KubernetesClusterSettings').customWindowsPackageURL]", + "localAKSeBinaryURL": "[steps('KubernetesClusterSettings').localAKSeBinaryURL]" } } -} +} \ No newline at end of file diff --git a/kubernetes/template/DeploymentTemplates/script.sh b/kubernetes/template/DeploymentTemplates/script.sh index 05797f4..d422487 100644 --- a/kubernetes/template/DeploymentTemplates/script.sh +++ b/kubernetes/template/DeploymentTemplates/script.sh @@ -1,9 +1,11 @@ -#!/bin/bash -e +#!/bin/bash -ex ERR_APT_INSTALL_TIMEOUT=9 # Timeout installing required apt packages ERR_AKSE_DOWNLOAD=10 # Failure downloading AKS-Engine binaries ERR_AKSE_DEPLOY=12 # Failure calling AKS-Engine's deploy operation ERR_TEMPLATE_DOWNLOAD=13 # Failure downloading AKS-Engine template +ERR_INVALID_AGENT_COUNT_VALUE=14 # Both Windows and Linux agent value is zero +ERR_TEMPLATE_GENERATION=15 # The default api model could not be generated ERR_CACERT_INSTALL=20 # Failure moving CA certificate ERR_METADATA_ENDPOINT=30 # Failure calling the metadata endpoint ERR_API_MODEL=40 # Failure building API model using user input @@ -17,7 +19,6 @@ ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete #ERR_AZS_LOGIN_ADFS=54 # Failure to log in to ADFS environment #ERR_AZS_ACCOUNT_SUB=55 # Failure setting account default subscription - function collect_deployment_and_operations { # Store main exit code @@ -34,6 +35,11 @@ function collect_deployment_and_operations # Collect deployment logs always, even if the script ends with an error trap collect_deployment_and_operations EXIT +cleanUpGPUDrivers() { + rm -f /etc/apt/sources.list.d/nvidia-docker.list + apt-key del $(apt-key list | grep NVIDIA -B 1 | head -n 1 | cut -d "/" -f 2 | cut -d " " -f 1) +} + ### # # Logs output by prepending date and log level type(Error, warning, info or verbose). @@ -110,13 +116,23 @@ ensure_certificates() # Download msazurestackworkloads' AKSe fork and move relevant files to the working directory download_akse() { - AKSE_ZIP_NAME="aks-engine-$AKSE_RELEASE_VERSION-linux-amd64" - AKSE_ZIP_URL="$AKSE_BASE_URL/$AKSE_RELEASE_VERSION/$AKSE_ZIP_NAME.tar.gz" + if [ ! $DISCONNECTED_AKS_ENGINE_URL ] + then + AKSE_ZIP_NAME="aks-engine-$AKSE_RELEASE_VERSION-linux-amd64" + AKSE_ZIP_URL="$AKSE_BASE_URL/$AKSE_RELEASE_VERSION/$AKSE_ZIP_NAME.tar.gz" + else + AKSE_ZIP_URL=$DISCONNECTED_AKS_ENGINE_URL + fi + log_level -i "AKSE_ZIP_URL:$AKSE_ZIP_URL" + curl --retry 5 --retry-delay 10 --max-time 60 -L -s -f -O $AKSE_ZIP_URL || exit $ERR_AKSE_DOWNLOAD - + mkdir -p ./bin - tar -xf $AKSE_ZIP_NAME.tar.gz - cp ./$AKSE_ZIP_NAME/aks-engine ./bin + AKSE_LOCAL_ZIP_NAME=${AKSE_ZIP_URL##*/} + tar -xf $AKSE_LOCAL_ZIP_NAME + AKSE_LOCAL_FILENAME=`basename -s .tar.gz $AKSE_LOCAL_ZIP_NAME` + cp ./$AKSE_LOCAL_FILENAME/aks-engine ./bin + AKSE_LOCATION=./bin/aks-engine if [ ! -f $AKSE_LOCATION ]; then @@ -125,9 +141,9 @@ download_akse() exit 1 fi - TEMPLATE_URL="https://raw.githubusercontent.com/$GALLERY_REPO/$GALLERY_BRANCH/kubernetes/template/DeploymentTemplates/$DEFINITION_TEMPLATE_NAME" - curl --retry 5 --retry-delay 10 --max-time 60 -s -f -O $TEMPLATE_URL || exit $ERR_TEMPLATE_DOWNLOAD + generate_api_model || exit $ERR_TEMPLATE_GENERATION + DEFINITION_TEMPLATE="./$DEFINITION_TEMPLATE_NAME" if [ ! -f $DEFINITION_TEMPLATE ]; then log_level -e "API model template for Kubernetes not found in expected location" @@ -208,10 +224,85 @@ apt_get_install() wait_for_apt_locks } +generate_api_model() +{ + touch $DEFINITION_TEMPLATE_NAME + cat > $DEFINITION_TEMPLATE_NAME < $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + log_level -i "Updating cluster definition done with Linux agent node details." +fi + +##################################################################################### +#Windows Agent +if [ "$WINDOWS_AGENT_COUNT" != "0" ]; then + log_level -i "Update cluster definition with Windows profile details." + + cat $AZURESTACK_CONFIGURATION | \ + jq --arg WINDOWS_ADMIN_USERNAME $WINDOWS_ADMIN_USERNAME '.properties.windowsProfile.adminUsername=$WINDOWS_ADMIN_USERNAME' | \ + jq --arg WINDOWS_ADMIN_PASSWORD $WINDOWS_ADMIN_PASSWORD '.properties.windowsProfile.adminPassword=$WINDOWS_ADMIN_PASSWORD' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + log_level -i "Update Windows agent node details." + + cat $AZURESTACK_CONFIGURATION | \ + jq --arg winAgentCount $WINDOWS_AGENT_COUNT --arg winAgentSize $WINDOWS_AGENT_SIZE --arg winAvailabilityProfile $AVAILABILITY_PROFILE \ + '.properties.agentPoolProfiles += [{"name": "windowspool", "osDiskSizeGB": 128, "AcceleratedNetworkingEnabled": false, "osType": "Windows", "count": $winAgentCount | tonumber, "vmSize": $winAgentSize, "availabilityProfile": $winAvailabilityProfile}]' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + log_level -i "Updating cluster definition done with Windows agent node details." +fi + +##################################################################################### +# custom windows package URL +if [ "$WINDOWS_CUSTOM_PACKAGE" != "" ]; then + log_level -i "Adding Windows custom package URL details." + + cat $AZURESTACK_CONFIGURATION | \ + jq --arg CUSTOM_PACKAGE $WINDOWS_CUSTOM_PACKAGE '.properties.orchestratorProfile.kubernetesConfig += {"customWindowsPackageURL": $CUSTOM_PACKAGE } ' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + log_level -i "Done updating Windows custom package URL details." +fi + +##################################################################################### +#custom vnet config +if [ "$CUSTOM_VNET_NAME" != "" ]; then + log_level -i "Setting general custom vnet properties." + MASTER_VNET_ID="/subscriptions/$TENANT_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Network/virtualNetworks/$CUSTOM_VNET_NAME/subnets/$MASTER_SUBNET_NAME" + AGENT_VNET_ID="/subscriptions/$TENANT_SUBSCRIPTION_ID/resourceGroups/$RESOURCE_GROUP_NAME/providers/Microsoft.Network/virtualNetworks/$CUSTOM_VNET_NAME/subnets/$AGENT_SUBNET_NAME" + + cat $AZURESTACK_CONFIGURATION | \ + jq --arg MASTER_VNET_ID $MASTER_VNET_ID '.properties.masterProfile += {"vnetSubnetId": $MASTER_VNET_ID } '| \ + jq --arg FIRST_CONSECUTIVE_STATIC_IP $FIRST_CONSECUTIVE_STATIC_IP '.properties.masterProfile += {"firstConsecutiveStaticIP": $FIRST_CONSECUTIVE_STATIC_IP } ' | \ + jq --arg AGENT_VNET_ID $AGENT_VNET_ID '.properties.agentPoolProfiles[0] += {"vnetSubnetId": $AGENT_VNET_ID } ' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + if [ "$WINDOWS_AGENT_COUNT" != "0" ]; then + + log_level -i "Updating custom vnet properties for Windows nodes." + if [ "$AGENT_COUNT" != "0" ]; then + cat $AZURESTACK_CONFIGURATION | \ + jq --arg AGENT_VNET_ID $AGENT_VNET_ID '.properties.agentPoolProfiles[1] += {"vnetSubnetId": $AGENT_VNET_ID } ' \ + > $AZURESTACK_CONFIGURATION_TEMP + else + cat $AZURESTACK_CONFIGURATION | \ + jq --arg AGENT_VNET_ID $AGENT_VNET_ID '.properties.agentPoolProfiles[0] += {"vnetSubnetId": $AGENT_VNET_ID } ' \ + > $AZURESTACK_CONFIGURATION_TEMP + fi + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + log_level -i "Custom vnet properties update for Windows nodes done ." + fi + + log_level -i "Done building custom vnet definition." +fi + +##################################################################################### +#custom network policy config +if [ "$NETWORK_POLICY" != "" ]; then + log_level -i "Setting network policy property." + cat $AZURESTACK_CONFIGURATION | jq --arg NETWORK_POLICY $NETWORK_POLICY '.properties.orchestratorProfile.kubernetesConfig.networkPolicy=$NETWORK_POLICY' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + + log_level -i "Done setting network policy property." +fi + +##################################################################################### +#tiller + +if [ "$ENABLE_TILLER" == "true" ]; then + log_level -i "Enabling Tiller Addon" + cat $AZURESTACK_CONFIGURATION | \ + jq --arg enableTiller $ENABLE_TILLER \ + '.properties.orchestratorProfile.kubernetesConfig.addons += [{"name": "tiller", "enabled": true}]' \ + > $AZURESTACK_CONFIGURATION_TEMP + + validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL +fi + ##################################################################################### # apimodel gen @@ -343,9 +585,6 @@ jq --arg ENDPOINT_PORTAL $ENDPOINT_PORTAL '.properties.customCloudProfile.portal jq --arg REGION_NAME $REGION_NAME '.location = $REGION_NAME' | \ jq --arg MASTER_DNS_PREFIX $MASTER_DNS_PREFIX '.properties.masterProfile.dnsPrefix = $MASTER_DNS_PREFIX' | \ jq --arg NODE_DISTRO $NODE_DISTRO '.properties.masterProfile.distro = $NODE_DISTRO' | \ -jq '.properties.agentPoolProfiles[0].count'=$AGENT_COUNT | \ -jq --arg AGENT_SIZE $AGENT_SIZE '.properties.agentPoolProfiles[0].vmSize=$AGENT_SIZE' | \ -jq --arg NODE_DISTRO $NODE_DISTRO '.properties.agentPoolProfiles[0].distro=$NODE_DISTRO' | \ jq '.properties.masterProfile.count'=$MASTER_COUNT | \ jq --arg MASTER_SIZE $MASTER_SIZE '.properties.masterProfile.vmSize=$MASTER_SIZE' | \ jq --arg ADMIN_USERNAME $ADMIN_USERNAME '.properties.linuxProfile.adminUsername = $ADMIN_USERNAME' | \ @@ -354,12 +593,15 @@ jq --arg AUTH_METHOD $AUTH_METHOD '.properties.customCloudProfile.authentication jq --arg SPN_CLIENT_ID $SPN_CLIENT_ID '.properties.servicePrincipalProfile.clientId = $SPN_CLIENT_ID' | \ jq --arg SPN_CLIENT_SECRET $SPN_CLIENT_SECRET '.properties.servicePrincipalProfile.secret = $SPN_CLIENT_SECRET' | \ jq --arg IDENTITY_SYSTEM_LOWER $IDENTITY_SYSTEM_LOWER '.properties.customCloudProfile.identitySystem=$IDENTITY_SYSTEM_LOWER' | \ -jq --arg K8S_VERSION $K8S_AZURE_CLOUDPROVIDER_VERSION '.properties.orchestratorProfile.orchestratorRelease=$K8S_VERSION' | \ -jq --arg K8S_IMAGE_BASE $K8S_IMAGE_BASE '.properties.orchestratorProfile.kubernetesConfig.kubernetesImageBase=$K8S_IMAGE_BASE' \ +jq --arg K8S_RELEASE $K8S_AZURE_CLOUDPROVIDER_RELEASE '.properties.orchestratorProfile.orchestratorRelease=$K8S_RELEASE' | \ +jq --arg K8S_VERSION $K8S_AZURE_CLOUDPROVIDER_VERSION '.properties.orchestratorProfile.orchestratorVersion=$K8S_VERSION' | \ +jq --arg NETWORK_PLUGIN $NETWORK_PLUGIN '.properties.orchestratorProfile.kubernetesConfig.networkPlugin=$NETWORK_PLUGIN' | \ +jq --arg CONTAINER_RUNTIME $CONTAINER_RUNTIME '.properties.orchestratorProfile.kubernetesConfig.containerRuntime=$CONTAINER_RUNTIME' \ > $AZURESTACK_CONFIGURATION_TEMP validate_and_restore_cluster_definition $AZURESTACK_CONFIGURATION_TEMP $AZURESTACK_CONFIGURATION || exit $ERR_API_MODEL + log_level -i "Done building cluster definition." ##################################################################################### diff --git a/kubernetes/template/manifest.json b/kubernetes/template/manifest.json index 2c71c24..c21dc61 100644 --- a/kubernetes/template/manifest.json +++ b/kubernetes/template/manifest.json @@ -2,7 +2,7 @@ "$schema": "https://gallery.azure.com/schemas/2015-10-01/manifest.json#", "name": "AzureStackKubernetesCluster", "publisher": "Microsoft", - "version": "0.5.1", + "version": "1.0.3", "displayName": "ms-resource:displayName", "publisherDisplayName": "ms-resource:publisherDisplayName", "publisherLegalName": "ms-resource:publisherDisplayName", @@ -57,7 +57,7 @@ { "displayName": "ms-resource:displayName", "publisherDisplayName": "ms-resource:publisherDisplayName", - "legalTerms": "

MICROSOFT SOFTWARE LICENSE TERMS<\/b> <\/p>

AKS ENGINE FOR AZURE STACK PREVIEW<\/b><\/p>

________________________________________<\/p>

These license terms are an agreement between you and Microsoft Corporation (or one of its affiliates). They apply to the software named above and any Microsoft services or software updates (except to the extent such services or updates are accompanied by new or additional terms, in which case those different terms apply prospectively and do not alter your or Microsoft\u2019s rights relating to pre-updated software or services). IF YOU COMPLY WITH THESE LICENSE TERMS, YOU HAVE THE RIGHTS BELOW. BY USING THE SOFTWARE, YOU ACCEPT THESE TERMS.<\/p>

1. INSTALLATION AND USE RIGHTS.<\/p>

a) General. You may install and use any number of copies of the software on your devices. You may not use the software in a live operating environment unless Microsoft permits you to do so under another agreement.<\/p>

b) Third Party Software. The software may include third party applications that Microsoft, not the third party, licenses to you under this agreement. Any included notices for third party applications are for your information only.<\/p>

c) Open Source Components. The software may contain third party copyrighted software licensed under open source licenses with source code availability obligations. Copies of those licenses are included in the ThirdPartyNotices file or other accompanying notices file. <\/p>

2. TIME-SENSITIVE SOFTWARE.<\/p>

a) Term. The term of this agreement is until 01\/06\/2019 (day\/month\/year).<\/p>

b) Notice. You may receive periodic reminder notices of this date through the software.<\/p>

c) Access to data. You may not be able to access data used in the software when it stops running.<\/p>

3. PRE-RELEASE SOFTWARE. The software is a pre-release version. It may not operate correctly. It may be different from the commercially released version.<\/p>

4. FEEDBACK. If you give feedback about the software to Microsoft, you give to Microsoft, without charge, the right to use, share and commercialize your feedback in any way and for any purpose. You will not give feedback that is subject to a license that requires Microsoft to license its software or documentation to third parties because Microsoft includes your feedback in them. These rights survive this agreement.<\/p>

5. DATA COLLECTION. The software may collect information about you and your use of the software and send that to Microsoft. Microsoft may use this information to provide services and improve Microsoft\u2019s products and services. Your opt-out rights, if any, are described in the product documentation. Some features in the software may enable collection of data from users of your applications that access or use the software. If you use these features to enable data collection in your applications, you must comply with applicable law, including getting any required user consent, and maintain a prominent privacy policy that accurately informs users about how you use, collect, and share their data. You can learn more about Microsoft\u2019s data collection and use in the product documentation and the Microsoft Privacy Statement at https:\/\/go.microsoft.com\/fwlink\/?LinkId=521839. You agree to comply with all applicable provisions of the Microsoft Privacy Statement.<\/p>

6. SCOPE OF LICENSE. The software is licensed, not sold. Microsoft reserves all other rights. Unless applicable law gives you more rights despite this limitation, you will not (and have no right to):<\/p>

a) work around any technical limitations in the software that only allow you to use it in certain ways;<\/p>

b) b)\treverse engineer, decompile, or disassemble the software, or attempt to do so, except and only to the extent required by third party licensing terms governing the use of certain open-source components that may be included with the software;<\/p>

c) remove, minimize, block, or modify any notices of Microsoft or its suppliers in the software;<\/p>

d) use the software for commercial, non-profit, or revenue-generating activities;<\/p>

e) use the software in any way that is against the law or to create or propagate malware; or<\/p>

f) share, publish, distribute, or lend the software, provide the software as a stand-alone hosted solution for others to use, or transfer the software or this agreement to any third party.<\/p>

7. EXPORT RESTRICTIONS. You must comply with all domestic and international export laws and regulations that apply to the software, which include restrictions on destinations, end users, and end use. For further information on export restrictions, visit http:\/\/aka.ms\/exporting<\/a>.<\/p>

8. SUPPORT SERVICES. Microsoft is not obligated under this agreement to provide any support services for the software. Any support provided is \u201Cas is\u201D, \u201Cwith all faults\u201D, and without warranty of any kind.<\/p>

9. UPDATES. The software may periodically check for updates, and download and install them for you. You may obtain updates only from Microsoft or authorized sources. Microsoft may need to update your system to provide you with updates. You agree to receive these automatic updates without any additional notice. Updates may not include or support all existing software features, services, or peripheral devices.<\/p>

10. TERMINATION. Without prejudice to any other rights, Microsoft may terminate this agreement if you fail to comply with any of its terms or conditions. In such event, you must destroy all copies of the software and all of its component parts.<\/p>

11. ENTIRE AGREEMENT. This agreement, and any other terms Microsoft may provide for supplements, updates, or third-party applications, is the entire agreement for the software.<\/p>

12. APPLICABLE LAW AND PLACE TO RESOLVE DISPUTES. If you acquired the software in the United States or Canada, the laws of the state or province where you live (or, if a business, where your principal place of business is located) govern the interpretation of this agreement, claims for its breach, and all other claims (including consumer protection, unfair competition, and tort claims), regardless of conflict of laws principles. If you acquired the software in any other country, its laws apply. If U.S. federal jurisdiction exists, you and Microsoft consent to exclusive jurisdiction and venue in the federal court in King County, Washington for all disputes heard in court. If not, you and Microsoft consent to exclusive jurisdiction and venue in the Superior Court of King County, Washington for all disputes heard in court.<\/p>

13. CONSUMER RIGHTS; REGIONAL VARIATIONS. This agreement describes certain legal rights. You may have other rights, including consumer rights, under the laws of your state, province, or country. Separate and apart from your relationship with Microsoft, you may also have rights with respect to the party from which you acquired the software. This agreement does not change those other rights if the laws of your state, province, or country do not permit it to do so. For example, if you acquired the software in one of the below regions, or mandatory country law applies, then the following provisions apply to you:<\/p>

a) Australia. You have statutory guarantees under the Australian Consumer Law and nothing in this agreement is intended to affect those rights.<\/p>

b) Canada. If you acquired this software in Canada, you may stop receiving updates by turning off the automatic update feature, disconnecting your device from the Internet (if and when you re-connect to the Internet, however, the software will resume checking for and installing updates), or uninstalling the software. The product documentation, if any, may also specify how to turn off updates for your specific device or software.<\/p>

c) Germany and Austria.<\/p>

i. Warranty. The properly licensed software will perform substantially as described in any Microsoft materials that accompany the software. However, Microsoft gives no contractual guarantee in relation to the licensed software.<\/p>

ii. Limitation of Liability. In case of intentional conduct, gross negligence, claims based on the Product Liability Act, as well as, in case of death or personal or physical injury, Microsoft is liable according to the statutory law.<\/p>

Subject to the foregoing clause ii., Microsoft will only be liable for slight negligence if Microsoft is in breach of such material contractual obligations, the fulfillment of which facilitate the due performance of this agreement, the breach of which would endanger the purpose of this agreement and the compliance with which a party may constantly trust in (so-called "cardinal obligations"). In other cases of slight negligence, Microsoft will not be liable for slight negligence.<\/p>

14. DISCLAIMER OF WARRANTY. THE SOFTWARE IS LICENSED \u201CAS IS.\u201D YOU BEAR THE RISK OF USING IT. MICROSOFT GIVES NO EXPRESS WARRANTIES, GUARANTEES, OR CONDITIONS. TO THE EXTENT PERMITTED UNDER APPLICABLE LAWS, MICROSOFT EXCLUDES ALL IMPLIED WARRANTIES, INCLUDING MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT.<\/p>

15. LIMITATION ON AND EXCLUSION OF DAMAGES. IF YOU HAVE ANY BASIS FOR RECOVERING DAMAGES DESPITE THE PRECEDING DISCLAIMER OF WARRANTY, YOU CAN RECOVER FROM MICROSOFT AND ITS SUPPLIERS ONLY DIRECT DAMAGES UP TO U.S. $5.00. YOU CANNOT RECOVER ANY OTHER DAMAGES, INCLUDING CONSEQUENTIAL, LOST PROFITS, SPECIAL, INDIRECT OR INCIDENTAL DAMAGES.<\/p>

This limitation applies to (a) anything related to the software, services, content (including code) on third party Internet sites, or third party applications; and (b) claims for breach of contract, warranty, guarantee, or condition; strict liability, negligence, or other tort; or any other claim; in each case to the extent permitted by applicable law.<\/p>

It also applies even if Microsoft knew or should have known about the possibility of the damages. The above limitation or exclusion may not apply to you because your state, province, or country may not allow the exclusion or limitation of incidental, consequential, or other damages.<\/p>

Please note: As this software is distributed in Canada, some of the clauses in this agreement are provided below in French.<\/p>

Remarque: Ce logiciel \u00E9tant distribu\u00E9 au Canada, certaines des clauses dans ce contrat sont fournies ci-dessous en fran\u00E7ais.<\/p>

EXON\u00C9RATION DE GARANTIE. Le logiciel vis\u00E9 par une licence est offert \u00AB tel quel \u00BB. Toute utilisation de ce logiciel est \u00E0 votre seule risque et p\u00E9ril. Microsoft n\u2019accorde aucune autre garantie expresse. Vous pouvez b\u00E9n\u00E9ficier de droits additionnels en vertu du droit local sur la protection des consommateurs, que ce contrat ne peut modifier. La ou elles sont permises par le droit locale, les garanties implicites de qualit\u00E9 marchande, d\u2019ad\u00E9quation \u00E0 un usage particulier et d\u2019absence de contrefa\u00E7on sont exclues.<\/p>

LIMITATION DES DOMMAGES-INT\u00C9R\u00CATS ET EXCLUSION DE RESPONSABILIT\u00C9 POUR LES DOMMAGES. Vous pouvez obtenir de Microsoft et de ses fournisseurs une indemnisation en cas de dommages directs uniquement \u00E0 hauteur de 5,00 $ US. Vous ne pouvez pr\u00E9tendre \u00E0 aucune indemnisation pour les autres dommages, y compris les dommages sp\u00E9ciaux, indirects ou accessoires et pertes de b\u00E9n\u00E9fices. Cette limitation concerne:<\/p>

\u2022 tout ce qui est reli\u00E9 au logiciel, aux services ou au contenu (y compris le code) figurant sur des sites Internet tiers ou dans des programmes tiers; et<\/p>

\u2022 les r\u00E9clamations au titre de violation de contrat ou de garantie, ou au titre de responsabilit\u00E9 stricte, de n\u00E9gligence ou d\u2019une autre faute dans la limite autoris\u00E9e par la loi en vigueur.<\/p>

Elle s\u2019applique \u00E9galement, m\u00EAme si Microsoft connaissait ou devrait conna\u00EEtre l\u2019\u00E9ventualit\u00E9 d\u2019un tel dommage. Si votre pays n\u2019autorise pas l\u2019exclusion ou la limitation de responsabilit\u00E9 pour les dommages indirects, accessoires ou de quelque nature que ce soit, il se peut que la limitation ou l\u2019exclusion ci-dessus ne s\u2019appliquera pas \u00E0 votre \u00E9gard.<\/p>

EFFET JURIDIQUE. Le pr\u00E9sent contrat d\u00E9crit certains droits juridiques. Vous pourriez avoir d\u2019autres droits pr\u00E9vus par les lois de votre pays. Le pr\u00E9sent contrat ne modifie pas les droits que vous conf\u00E8rent les lois de votre pays si celles-ci ne le permettent pas.<\/p>", + "legalTerms": "

MICROSOFT SOFTWARE LICENSE TERMS<\/b> <\/p>

MICROSOFT SOFTWARE LICENSE TERMS KUBERNETES CLUSTER USING AKS ENGINE <\/b><\/p>

________________________________________<\/p>

These license terms are an agreement between you and Microsoft Corporation (or one of its affiliates). They apply to the software named above and any Microsoft services or software updates (except to the extent such services or updates are accompanied by new or additional terms, in which case those different terms apply prospectively and do not alter your or Microsoft’s rights relating to pre-updated software or services). IF YOU COMPLY WITH THESE LICENSE TERMS, YOU HAVE THE RIGHTS BELOW. BY USING THE SOFTWARE, YOU ACCEPT THESE TERMS.<\/p>

1. INSTALLATION AND USE RIGHTS.<\/p>

a) General. You may install and use any number of copies of the software to develop and test your applications. You may not use the software in a production environment (you may use it in development / test / evaluation environments) unless Microsoft permits you to do so under another agreement.<\/p>

b) Third Party Software. The software may include third party applications that Microsoft, not the third party, licenses to you under this agreement. Any included notices for third party applications are for your information only.<\/p>

2. DATA COLLECTION. The software may collect information about you and your use of the software and send that to Microsoft. Microsoft may use this information to provide services and improve Microsoft’s products and services. Your opt-out rights, if any, are described in the product documentation. Some features in the software may enable collection of data from users of your applications that access or use the software. If you use these features to enable data collection in your applications, you must comply with applicable law, including getting any required user consent, and maintain a prominent privacy policy that accurately informs users about how you use, collect, and share their data. You can learn more about Microsoft’s data collection and use in the product documentation and the Microsoft Privacy Statement at http:\/\/aka.ms\/privacy<\/a>. You agree to comply with all applicable provisions of the Microsoft Privacy Statement.<\/p>3. SCOPE OF LICENSE. The software is licensed, not sold. Microsoft reserves all other rights. Unless applicable law gives you more rights despite this limitation, you will not (and have no right to):<\/p>

a) work around any technical limitations in the software that only allow you to use it in certain ways;<\/p>

b) reverse engineer, decompile or disassemble the software;<\/p>

c) remove, minimize, block, or modify any notices of Microsoft or its suppliers in the software;<\/p>

d) use the software for commercial, non-profit, or revenue-generating activities;<\/p>

e) use the software in any way that is against the law or to create or propagate malware; or<\/p>

f) share, publish, distribute, or lend the software, provide the software as a stand-alone hosted solution for others to use, or transfer the software or this agreement to any third party.<\/p>

4. EXPORT RESTRICTIONS. You must comply with all domestic and international export laws and regulations that apply to the software, which include restrictions on destinations, end users, and end use. For further information on export restrictions, visit http:\/\/aka.ms\/exporting<\/a><\/p>

5. SUPPORT SERVICES. Microsoft is not obligated under this agreement to provide any support services for the software. Any support provided is “as is”, “with all faults”, and without warranty of any kind.<\/p>

6. UPDATES. The software may periodically check for updates, and download and install them for you. You may obtain updates only from Microsoft or authorized sources. Microsoft may need to update your system to provide you with updates. You agree to receive these automatic updates without any additional notice. Updates may not include or support all existing software features, services, or peripheral devices.<\/p>

7. TERMINATION. Without prejudice to any other rights, Microsoft may terminate this agreement if you fail to comply with any of its terms or conditions. In such event, you must destroy all copies of the software and all of its component parts. <\/p>

8. ENTIRE AGREEMENT. This agreement, and any other terms Microsoft may provide for supplements, updates, or third-party applications, is the entire agreement for the software.<\/p>

9. APPLICABLE LAW AND PLACE TO RESOLVE DISPUTES. If you acquired the software in the United States or Canada, the laws of the state or province where you live (or, if a business, where your principal place of business is located) govern the interpretation of this agreement, claims for its breach, and all other claims (including consumer protection, unfair competition, and tort claims), regardless of conflict of laws principles. If you acquired the software in any other country, its laws apply. If U.S. federal jurisdiction exists, you and Microsoft consent to exclusive jurisdiction and venue in the federal court in King County, Washington for all disputes heard in court. If not, you and Microsoft consent to exclusive jurisdiction and venue in the Superior Court of King County, Washington for all disputes heard in court.<\/p>

10. CONSUMER RIGHTS; REGIONAL VARIATIONS. This agreement describes certain legal rights. You may have other rights, including consumer rights, under the laws of your state, province, or country. Separate and apart from your relationship with Microsoft, you may also have rights with respect to the party from which you acquired the software. This agreement does not change those other rights if the laws of your state, province, or country do not permit it to do so. For example, if you acquired the software in one of the below regions, or mandatory country law applies, then the following provisions apply to you:<\/p>

a) Australia. You have statutory guarantees under the Australian Consumer Law and nothing in this agreement is intended to affect those rights.<\/p>

b) Canada. If you acquired this software in Canada, you may stop receiving updates by turning off the automatic update feature, disconnecting your device from the Internet (if and when you re-connect to the Internet, however, the software will resume checking for and installing updates), or uninstalling the software. The product documentation, if any, may also specify how to turn off updates for your specific device or software.<\/p>

c) Germany and Austria.<\/p>

i. Warranty. The properly licensed software will perform substantially as described in any Microsoft materials that accompany the software. However, Microsoft gives no contractual guarantee in relation to the licensed software.<\/p>

Limitation of Liability. In case of intentional conduct, gross negligence, claims based on the Product Liability Act, as well as, in case of death or personal or physical injury, Microsoft is liable according to the statutory law.<\/p>

Subject to the foregoing clause ii., Microsoft will only be liable for slight negligence if Microsoft is in breach of such material contractual obligations, the fulfillment of which facilitate the due performance of this agreement, the breach of which would endanger the purpose of this agreement and the compliance with which a party may constantly trust in (so-called cardinal obligations). In other cases of slight negligence, Microsoft will not be liable for slight negligence.<\/p>

11. DISCLAIMER OF WARRANTY. THE SOFTWARE IS LICENSED “AS IS.” YOU BEAR THE RISK OF USING IT. MICROSOFT GIVES NO EXPRESS WARRANTIES, GUARANTEES, OR CONDITIONS. TO THE EXTENT PERMITTED UNDER APPLICABLE LAWS, MICROSOFT EXCLUDES ALL IMPLIED WARRANTIES, INCLUDING MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND NON-INFRINGEMENT.<\/p>

12. LIMITATION ON AND EXCLUSION OF DAMAGES. IF YOU HAVE ANY BASIS FOR RECOVERING DAMAGES DESPITE THE PRECEDING DISCLAIMER OF WARRANTY, YOU CAN RECOVER FROM MICROSOFT AND ITS SUPPLIERS ONLY DIRECT DAMAGES UP TO U.S. $5.00. YOU CANNOT RECOVER ANY OTHER DAMAGES, INCLUDING CONSEQUENTIAL, LOST PROFITS, SPECIAL, INDIRECT OR INCIDENTAL DAMAGES.<\/p>

This limitation applies to (a) anything related to the software, services, content (including code) on third party Internet sites, or third party applications; and (b) claims for breach of contract, warranty, guarantee, or condition; strict liability, negligence, or other tort; or any other claim; in each case to the extent permitted by applicable law.<\/p>

It also applies even if Microsoft knew or should have known about the possibility of the damages. The above limitation or exclusion may not apply to you because your state, province, or country may not allow the exclusion or limitation of incidental, consequential, or other damages.<\/p>

Please note: As this software is distributed in Canada, some of the clauses in this agreement are provided below in French.<\/p>

Please note: As this software is distributed in Canada, some of the clauses in this agreement are provided below in French.<\/p>

Remarque: Ce logiciel \u00E9tant distribu\u00E9 au Canada, certaines des clauses dans ce contrat sont fournies ci-dessous en fran\u00E7ais.<\/p>

EXON\u00C9RATION DE GARANTIE. Le logiciel vis\u00E9 par une licence est offert \u00AB tel quel \u00BB. Toute utilisation de ce logiciel est \u00E0 votre seule risque et p\u00E9ril. Microsoft n\u2019accorde aucune autre garantie expresse. Vous pouvez b\u00E9n\u00E9ficier de droits additionnels en vertu du droit local sur la protection des consommateurs, que ce contrat ne peut modifier. La ou elles sont permises par le droit locale, les garanties implicites de qualit\u00E9 marchande, d\u2019ad\u00E9quation \u00E0 un usage particulier et d\u2019absence de contrefa\u00E7on sont exclues.<\/p>

LIMITATION DES DOMMAGES-INT\u00C9R\u00CATS ET EXCLUSION DE RESPONSABILIT\u00C9 POUR LES DOMMAGES. Vous pouvez obtenir de Microsoft et de ses fournisseurs une indemnisation en cas de dommages directs uniquement \u00E0 hauteur de 5,00 $ US. Vous ne pouvez pr\u00E9tendre \u00E0 aucune indemnisation pour les autres dommages, y compris les dommages sp\u00E9ciaux, indirects ou accessoires et pertes de b\u00E9n\u00E9fices. Cette limitation concerne:<\/p>

\u2022 tout ce qui est reli\u00E9 au logiciel, aux services ou au contenu (y compris le code) figurant sur des sites Internet tiers ou dans des programmes tiers; et<\/p>

\u2022 les r\u00E9clamations au titre de violation de contrat ou de garantie, ou au titre de responsabilit\u00E9 stricte, de n\u00E9gligence ou d\u2019une autre faute dans la limite autoris\u00E9e par la loi en vigueur.<\/p>

Elle s\u2019applique \u00E9galement, m\u00EAme si Microsoft connaissait ou devrait conna\u00EEtre l\u2019\u00E9ventualit\u00E9 d\u2019un tel dommage. Si votre pays n\u2019autorise pas l\u2019exclusion ou la limitation de responsabilit\u00E9 pour les dommages indirects, accessoires ou de quelque nature que ce soit, il se peut que la limitation ou l\u2019exclusion ci-dessus ne s\u2019appliquera pas \u00E0 votre \u00E9gard.<\/p>

EFFET JURIDIQUE. Le pr\u00E9sent contrat d\u00E9crit certains droits juridiques. Vous pourriez avoir d\u2019autres droits pr\u00E9vus par les lois de votre pays. Le pr\u00E9sent contrat ne modifie pas les droits que vous conf\u00E8rent les lois de votre pays si celles-ci ne le permettent pas.<\/p>", "privacyPolicy": "https://privacy.microsoft.com/en-us/privacystatement", "offerDetails": { "publisherId": "msk8sazurestack", diff --git a/kubernetes/template/strings/resources.resjson b/kubernetes/template/strings/resources.resjson index b1ac2bf..caee7ba 100644 --- a/kubernetes/template/strings/resources.resjson +++ b/kubernetes/template/strings/resources.resjson @@ -1,9 +1,9 @@ { "displayName": "Kubernetes Cluster", "publisherDisplayName": "Microsoft", - "summary": "An IaaS based template for deploying a standalone Kubernetes Cluster", - "longSummary": "This solution deploys Kubernetes cluster running as an standalone cluster with templates generated using AKS Engine.", - "description": "This solution deploys a Kubernetes cluster running as a standalone cluster with templates generated using AKS Engine.

Kubernetes is a distributed system platform used to build scalable, reliable, and easily-managed applications for the cloud. You can use Kubernetes to:

• Develop upgradeable and scalable applications using Helm that can be deployed in seconds.
• Easily monitor and diagnose the health of your applications.", + "summary": "An IaaS based template for deploying a standalone Kubernetes cluster", + "longSummary": "This solution deploys a standalone Kubernetes cluster using AKS Engine.", + "description": "This solution deploys a standalone Kubernetes cluster using AKS Engine.

This solution is intended to deploy clusters as a proof-of-concept. For supported Kubernetes clusters on Azure Stack, use the AKS engine.

Kubernetes is a distributed system platform used to build scalable, reliable, and easily-managed applications for the cloud.

", "documentationLink": "Kubernetes Help", "documentationLink2": "Add a Kubernetes Cluster to the Azure Stack Marketplace", "documentationLink3": "Deploy a Kubernetes cluster to Azure Stack", diff --git a/registry/ContainerRegistryLegal-Release.docx b/registry/ContainerRegistryLegal-Release.docx new file mode 100644 index 0000000..a6a5c2e Binary files /dev/null and b/registry/ContainerRegistryLegal-Release.docx differ diff --git a/registry/ContainerRegistryLegal-Release.html b/registry/ContainerRegistryLegal-Release.html new file mode 100644 index 0000000..cffdbfb --- /dev/null +++ b/registry/ContainerRegistryLegal-Release.html @@ -0,0 +1,120 @@ +

MICROSOFT SOFTWARE LICENSE TERMS

+

CONTAINER REGISTRY TEMPLATE

+ +

These license terms are an agreement between you and Microsoft Corporation (or one of its affiliates). They + apply to the software named above and any Microsoft services or software updates (except to the extent such + services or updates are accompanied by new or additional terms, in which case those different terms apply + prospectively and do not alter your or Microsoft’s rights relating to pre-updated software or services). IF YOU + COMPLY WITH THESE LICENSE TERMS, YOU HAVE THE RIGHTS BELOW. BY USING THE SOFTWARE, YOU ACCEPT THESE TERMS. +

+

1. INSTALLATION AND USE RIGHTS.

+

a) General. You may install and use any number of copies of the software on your devices.

+

b) Third Party Software. The software may include third party applications that Microsoft, not the third + party, licenses to you under this agreement. Any included notices for third party applications are for your + information only.

+

2. DATA COLLECTION. The software may collect information about you and your use of the software and send + that to Microsoft. Microsoft may use this information to provide services and improve Microsoft’s products and + services. Your opt-out rights, if any, are described in the product documentation. Some features in the software + may enable collection of data from users of your applications that access or use the software. If you use these + features to enable data collection in your applications, you must comply with applicable law, including getting + any required user consent, and maintain a prominent privacy policy that accurately informs users about how you + use, collect, and share their data. You can learn more about Microsoft’s data collection and use in the product + documentation and the Microsoft Privacy Statement at http://aka.ms/privacy. + You agree to comply with all applicable provisions of the Microsoft Privacy Statement.

+

3. SCOPE OF LICENSE. The software is licensed, not sold. Microsoft reserves all other rights. Unless + applicable law gives you more rights despite this limitation, you will not (and have no right to):

+

a) work around any technical limitations in the software that only allow you to use it in certain ways; +

+

b) reverse engineer, decompile or disassemble the software;

+

c) remove, minimize, block, or modify any notices of Microsoft or its suppliers in the software;

+

d) use the software for commercial, non-profit, or revenue-generating activities;

+

e) use the software in any way that is against the law or to create or propagate malware; or

+

f) share, publish, distribute, or lend the software, provide the software as a stand-alone hosted + solution for others to use, or transfer the software or this agreement to any third party.

+

4. EXPORT RESTRICTIONS. You must comply with all domestic and international export laws and regulations + that apply to the software, which include restrictions on destinations, end users, and end use. For further + information on export restrictions, visit http://aka.ms/exporting.

+

5. SUPPORT SERVICES. Microsoft is not obligated under this agreement to provide any support services for + the software. Any support provided is “as is”, “with all faults”, and without warranty of any kind.

+

6. UPDATES. The software may periodically check for updates, and download and install them for you. You + may obtain updates only from Microsoft or authorized sources. Microsoft may need to update your system to provide + you with updates. You agree to receive these automatic updates without any additional notice. Updates may not + include or support all existing software features, services, or peripheral devices.

+

7. ENTIRE AGREEMENT. This agreement, and any other terms Microsoft may provide for supplements, updates, + or third-party applications, is the entire agreement for the software.

+

8. APPLICABLE LAW AND PLACE TO RESOLVE DISPUTES. If you acquired the software in the United States or + Canada, the laws of the state or province where you live (or, if a business, where your principal place of + business is located) govern the interpretation of this agreement, claims for its breach, and all other claims + (including consumer protection, unfair competition, and tort claims), regardless of conflict of laws principles. + If you acquired the software in any other country, its laws apply. If U.S. federal jurisdiction exists, you and + Microsoft consent to exclusive jurisdiction and venue in the federal court in King County, Washington for all + disputes heard in court. If not, you and Microsoft consent to exclusive jurisdiction and venue in the Superior + Court of King County, Washington for all disputes heard in court.

+

9. CONSUMER RIGHTS; REGIONAL VARIATIONS. This agreement describes certain legal rights. You may + have other rights, including consumer rights, under the laws of your state, province, or country. Separate and + apart from your relationship with Microsoft, you may also have rights with respect to the party from which you + acquired the software. This agreement does not change those other rights if the laws of your state, province, or + country do not permit it to do so. For example, if you acquired the software in one of the below regions, or + mandatory country law applies, then the following provisions apply to you:

+

a) Australia. You have statutory guarantees under the Australian Consumer Law and nothing in this + agreement is intended to affect those rights.

+

b) Canada. If you acquired this software in Canada, you may stop receiving updates by turning off the + automatic update feature, disconnecting your device from the Internet (if and when you re-connect to the + Internet, however, the software will resume checking for and installing updates), or uninstalling the software. + The product documentation, if any, may also specify how to turn off updates for your specific device or + software. +

+

c) Germany and Austria.

+

i. Warranty. The properly licensed software will perform substantially as described in any Microsoft + materials that accompany the software. However, Microsoft gives no contractual guarantee in relation to the + licensed software.

+

ii. Limitation of Liability. In case of intentional conduct, gross negligence, claims based on the + Product Liability Act, as well as, in case of death or personal or physical injury, Microsoft is liable + according to the statutory law.

+

Subject to the foregoing clause ii., Microsoft will only be liable for slight negligence if Microsoft is in + breach of such material contractual obligations, the fulfillment of which facilitate the due performance of this + agreement, the breach of which would endanger the purpose of this agreement and the compliance with which a + party may constantly trust in (so-called "cardinal obligations"). In other cases of slight negligence, Microsoft + will not be liable for slight negligence.

+

10. DISCLAIMER OF WARRANTY. THE SOFTWARE IS LICENSED “AS IS.” YOU BEAR THE RISK OF USING IT. MICROSOFT GIVES + NO EXPRESS WARRANTIES, GUARANTEES, OR CONDITIONS. TO THE EXTENT PERMITTED UNDER APPLICABLE LAWS, MICROSOFT + EXCLUDES ALL IMPLIED WARRANTIES, INCLUDING MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, AND + NON-INFRINGEMENT.

+

11. LIMITATION ON AND EXCLUSION OF DAMAGES. IF YOU HAVE ANY BASIS FOR RECOVERING DAMAGES DESPITE THE + PRECEDING DISCLAIMER OF WARRANTY, YOU CAN RECOVER FROM MICROSOFT AND ITS SUPPLIERS ONLY DIRECT DAMAGES UP TO + U.S. $5.00. YOU CANNOT RECOVER ANY OTHER DAMAGES, INCLUDING CONSEQUENTIAL, LOST PROFITS, SPECIAL, INDIRECT + OR INCIDENTAL DAMAGES.

+

This limitation applies to (a) anything related to the software, services, content (including code) on third + party Internet sites, or third party applications; and (b) claims for breach of contract, warranty, + guarantee, or condition; strict liability, negligence, or other tort; or any other claim; in each case to + the extent permitted by applicable law.

+

It also applies even if Microsoft knew or should have known about the possibility of the damages. The above + limitation or exclusion may not apply to you because your state, province, or country may not allow the + exclusion or limitation of incidental, consequential, or other damages.

+

Please note: As this software is distributed in Canada, some of the clauses in this agreement are provided + below in French.

+

Remarque: Ce logiciel étant distribué au Canada, certaines des clauses dans ce contrat sont fournies + ci-dessous en français.

+

EXONÉRATION DE GARANTIE. Le logiciel visé par une licence est offert « tel quel ». Toute utilisation de ce + logiciel est à votre seule risque et péril. Microsoft n’accorde aucune autre garantie expresse. Vous pouvez + bénéficier de droits additionnels en vertu du droit local sur la protection des consommateurs, que ce + contrat ne peut modifier. La ou elles sont permises par le droit locale, les garanties implicites de qualité + marchande, d’adéquation à un usage particulier et d’absence de contrefaçon sont exclues.

+

LIMITATION DES DOMMAGES-INTÉRÊTS ET EXCLUSION DE RESPONSABILITÉ POUR LES DOMMAGES. Vous pouvez obtenir de + Microsoft et de ses fournisseurs une indemnisation en cas de dommages directs uniquement à hauteur de 5,00 $ + US. Vous ne pouvez prétendre à aucune indemnisation pour les autres dommages, y compris les dommages + spéciaux, indirects ou accessoires et pertes de bénéfices.

+

Cette limitation concerne:

+
    +
  • tout ce qui est relié au logiciel, aux services ou au contenu (y compris le code) figurant sur des + sites Internet tiers ou dans des programmes tiers; et
    • +
  • les réclamations au titre de violation de contrat ou de garantie, ou au titre de responsabilité + stricte, de négligence ou d’une autre faute dans la limite autorisée par la loi en vigueur.
    • +
+

Elle s’applique également, même si Microsoft connaissait ou devrait connaître l’éventualité d’un tel dommage. + Si votre pays n’autorise pas l’exclusion ou la limitation de responsabilité pour les dommages indirects, + accessoires ou de quelque nature que ce soit, il se peut que la limitation ou l’exclusion ci-dessus ne + s’appliquera pas à votre égard.

+

EFFET JURIDIQUE. Le présent contrat décrit certains droits juridiques. Vous pourriez avoir d’autres droits + prévus par les lois de votre pays. Le présent contrat ne modifie pas les droits que vous confèrent les lois + de votre pays si celles-ci ne le permettent pas.

\ No newline at end of file diff --git a/registry/DeploymentTemplates/azuredeploy.json b/registry/DeploymentTemplates/azuredeploy.json new file mode 100644 index 0000000..49997a9 --- /dev/null +++ b/registry/DeploymentTemplates/azuredeploy.json @@ -0,0 +1,334 @@ +{ + "$schema": "http://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "adminUsername": { + "type": "string", + "defaultValue": "azureuser", + "metadata": { + "description": "The linux user name." + } + }, + "adminPublicKey": { + "type": "string", + "metadata": { + "description": "User's RSA public key." + } + }, + "virtualMachineSize": { + "type": "string", + "defaultValue": "Standard_F8s_v2", + "metadata": { + "description": "The VM size." + } + }, + "virtualMachinePublisher": { + "type": "string", + "defaultValue": "microsoft-aks", + "metadata": { + "description": "The guest OS image publisher." + } + }, + "virtualMachineOffer": { + "type": "string", + "defaultValue": "aks", + "metadata": { + "description": "The guest OS image offer." + } + }, + "virtualMachineSku": { + "type": "string", + "defaultValue": "aks-ubuntu-1604-201910", + "metadata": { + "description": "The guest OS image SKU." + } + }, + "virtualMachineVersion": { + "type": "string", + "defaultValue": "latest", + "metadata": { + "description": "The guest OS image version." + } + }, + "pipName": { + "type": "string", + "metadata": { + "description": "The public IP resource name." + } + }, + "pipDomainNameLabel": { + "type": "string", + "metadata": { + "description": "The public IP DNS label." + } + }, + "pipAllocationMethod": { + "type": "string", + "defaultValue": "dynamic", + "metadata": { + "description": "The public IP allocation method." + } + }, + "storageAccountResourceId": { + "type": "string", + "metadata": { + "description": "An already existing storage account resource identifier." + } + }, + "storageAccountContainer": { + "type": "string", + "metadata": { + "description": "An already existing storage account container name." + } + }, + "pfxKeyVaultResourceId": { + "type": "string", + "metadata": { + "description": "The Key Vault resource identifier." + } + }, + "pfxKeyVaultSecretUrl": { + "type": "string", + "metadata": { + "description": "Absolute URL to the Key Vault secret that stores the pfx certificate." + } + }, + "pfxThumbprint": { + "type": "string", + "metadata": { + "description": "The certificate thumbprint." + } + }, + "registryReplicas": { + "type": "string", + "defaultValue": "20", + "metadata": { + "description": "Registry replicas." + } + }, + "servicePrincipalClientId": { + "type": "securestring", + "metadata": { + "description": "Client ID with access to list and get secrets from the credentials Key Vault instance" + } + }, + "servicePrincipalClientSecret": { + "type": "securestring", + "metadata": { + "description": "Secret of the client with access to list and get secrets from the credentials Key Vault instance" + } + }, + "enableValidations": { + "type": "string", + "defaultValue": "false", + "metadata": { + "description": "Registry validation flag." + } + } + }, + "variables": { + "rgname": "[resourceGroup().name]", + "nsgName": "[concat(variables('rgname'), '-nsg')]", + "nicName": "[concat(variables('rgname'), '-nic')]", + "vnetName": "[concat(variables('rgname'), '-vnet')]", + "vnetId": "[resourceId('Microsoft.Network/virtualNetworks',variables('vnetName'))]", + "subnetRef": "[concat(variables('vnetId'), '/subnets/default')]", + "tenantId": "[subscription().tenantId]", + "location": "[resourceGroup().location]", + "marketplaceVersion": "1.0.3", + "registryTag": "2.7.1", + "containerStatusWaitTimeInSeconds": "900", + "provisionScriptParameters": "[concat('MARKETPLACE_VERSION=', variables('marketplaceVersion'),' CONTAINER_STATUS_WAIT_TIME=', variables('containerStatusWaitTimeInSeconds'),' ADMIN_USER_NAME=', parameters('adminUsername'),' SA_RESOURCE_ID=', parameters('storageAccountResourceId'),' SA_CONTAINER=', parameters('storageAccountContainer'),' KV_RESOURCE_ID=', parameters('pfxKeyVaultResourceId'),' CERT_THUMBPRINT=', parameters('pfxThumbprint'),' PIP_LABEL=', parameters('pipDomainNameLabel'),' REGISTRY_TAG=', variables('registryTag'),' SPN_CLIENT_ID=',parameters('servicePrincipalClientId'),' SPN_CLIENT_SECRET=',parameters('servicePrincipalClientSecret'),' ENABLE_VALIDATIONS=',parameters('enableValidations'),' REGISTRY_REPLICAS=', parameters('registryReplicas'))]" + }, + "resources": [ + { + "type": "Microsoft.Compute/virtualMachines", + "name": "[concat(variables('rgname'),'-vm')]", + "apiVersion": "2017-03-30", + "location": "[resourceGroup().location]", + "properties": { + "osProfile": { + "computerName": "[concat(variables('rgname'),'-vm')]", + "adminUsername": "[parameters('adminUsername')]", + "customData": "[base64(concat('#cloud-config\n\nwrite_files:\n- path: \"/opt/azure/containers/script.sh\"\n permissions: \"0744\"\n encoding: gzip\n owner: \"root\"\n content: !!binary |\n H4sIAAAAAAAA/+w7f3PbtpL/81NsWU5t946kZMd5c8pj3tEW7WgiSz6JSl4vzXFgEpJYUQQDgHZcWd/9BuBvicqvZqZ3b9rOuBSwu1jsLnYXi+2PP5h3YWzeIbZUFGcy8exb1xuMpq49HHru4MYZz1zrP6D650dwwzUmKYcwZhxFURgvgOIPaUhxACjhkCB/hRaYSXI3g+l0MLr2LieudzUYOla3A9Agd4EC8DHlwJfp+i6hYcxhPIFk/hFiIlaBFX6Ee5RGcpzjdRIhjmEdMp/E83CRioXf3ADDPsWcif/ykMSN5V87v+TLd/+M5WdTZ+JdTpy+M3IH9nBqdU/z5UcEUoYp+BQHOOYhilhOCeYkjQMgMSzCe1xjQ1KeONeDqTv5xRuNXW8yG40Go2ure1ZX0xKDT2KOwhhToHgRMk4fYY7CCAfACTCOKAeW+j5mbJ5G0WPG8/jiF2kFw8HULU3g9HzfAB5QyIX254TCmtw9Su0zklK/UP7Uu77NhN8fvx0Nx3a/Ivj8EKmbKVzfXsv9BuQhjggKKsZ2TfP0by2WucdYbqoZGce1+7ZrW2dNS6zIOJQSCh9STB8FlTXmKEAcFVu6nYz7Xt+52N/Ts9MDTCw5T1jPNIujYaxDnxJG5tzwydrMDMlM79KYp2b3udF5VoLqJaieUBIYAb7bY+T29bVn9/velT0YWs/O8m1clZpGQQAUJwSS1QLmYYSbJjQcXw9GEtnpW+edShAVhYgsIIw5Ab7EEBB/VTOpJrHb2fRVSasLbcSSlC0hXIvdwZeRHA5LkqcHSEZRQXJOyfrzRC9mg2G/pHrWRvQuDaMgowoR8VFxQkoaE+dm/MYpiTxroUHxmtzjFiLihM1u+7brVG623c/WDQklXF9gDmkSCCfECfhknUSYY0XxI4ziWXJ9O+vT8B5TdnwCGwWArkGfzEG7vp15fWfq5kNzMDH3TZRwMz+zRhQybgRmfB8GIdIz8clBBeTK8kjiCLTj4peYhCdYUJzA6M2gP7BBv4AuPMESowD0WH77KQc9ANVUxbKntRGQI90TZasoFHP66K8DL5x7wkulFOc7ABBzIWaW1n0h5eGxCOPE0k5fAM/EZGlnL0DCsmU45/DTTzsfck7IMBROXTtm+AN0Qcspn7yAgCiF8HOaoJUfm//cCkp3FKMVPD3BryVsOId3oIWg4w8lNXj/QlhgrNQdC8U8pTF0y0EcMdyAkJsCrdpgOTsP5WdAYqxsFQkwJ9RDCfci4q9YKaiHZRhhmMuYYt4jakbhnRkkq4Up4KohoXahPFYb95G/xHIGUX8Z3uN88qUZ4HszFkfs9OVP3YaksL8kcPS2ZqIURxgxDGQu44Fk70hpbvGsvhuxiQXmXmbTeyrvduRPAZVBeCTlScotk68TMz8QejZlkPSr9LwvyHJKyAx0vYzxoKNyrjiF+hz0KrgUsz/A8c4xFVKDJ+AYg7a3j+L46A6o/3P87q3zvmf8fPJ0/A477yk1fj7R1BNhepXF+Yi30Wk3z3bg72S9uTrPW+30gIArheeC+7MO+R9Ufs496AT6yWrR640TkfexXs9SdX1OqI8lARIFKuh6TPQcQ6fYJ+s1jgMmzOfPdi3F4aqO4NdpM99VHyeVF5pdzEbuzJs4Q8eeOpZ2HLE7r3AMOgWdnUg4mdK9cSbTwXhkqWdGx3iuKqUt7AQD6J524BxOz8FPafR1OZW2abK0NWUyJcPXS5CeZCfPyiLbE+CPIQftYGJ8mNkOwLn46yeH6R+KwH9o4a+R0go/soozAzFfuKNEGH+AEV0TuiceQ0w3+Tuc5n+DeCT9UjCcpozjQAzuy+Uz6zatuom7n35JlAZeccJPO3DWkXJFiQiRp3rKw4jJq4WO40UYY0vb1G15m835Ubg3oesoisiDLq42C4oCzFqUvXPLkVyJkL4mAejoushstY3dvxmMsuvlyL5xttUm8mzRyxUv3KE0iN8+1JxgOIcf9rYr9grPO50sbOT4O07nN5LSGEU+jzLXlqAFpqCnDZzKGzXE3ra53Qxnjrm/HNspX5Y+xRn1b8eDkTu1tOP1StzDMx8id6Uz0HVpZ3BefOkBjtCjsDFdX6OPuogh8LwjonblYovzsUYxWuA1jrmhba7+qz/amsW9z8RxkJAw5uwfKAl1kVmHJLZOO91zvdPVO114CdqmZG+bObDMd/8D9BhDZ89p1/SdX0eVmhDkn7E9c19Z2vFvH4THNFAqCPDQRyLKGBFZhLGTM9ZYPpOKzMu0jSSyLVMMDKqJgjnT1B0m8YcWJt3xa2fkzSZDSy0omUTwcWpyssKxquyFmhYUbeM6I3vkeoN+G/o8LNR9WZVASqVPnOl4Nrl0DsoBpUGIYx+zd533DTHAEzAcgMqeWlRcjsmCivGknmTykOxb2vG32JT+T7gdT92abemvQL0kMccx193HBPcAJUmUc25+1B8eHkSesNZTGuHYJwEO1Dp6AOqCoph7/DHBlh+FOOZerVC0C5xDhIGlbaa3I+9yOHAyuTcgdWHU1ZolXlZ2auJOncuJ434Gn+IsfFnaptBXA0PblGYh1JIp8shAsvDkSVs4yhXw+k1mPIV+tM3rN5lnMzJVFYczr7ZlNpRxOf1GxVWMqnI5webuSX+udzt6p6tKnQq3RGj4u9RjDy4wotIZy11u1XKLxj2KUvzuvREG+fboGowlTxBjD4Ec4CT1lztjwnPnJUCRw27y3W1z/wglu9mEFE+VRGmbDNc0tXwvahuCpR3nLqKFyra6oR+ZR/LOfrJL5I09nDl/WOK51DOWv6fUT3aS6M2Pdb63oEcYTs/PW7PpQhugX9xVummKalv9zCgeTr2lnNXpKkwScUfeIYPKeq/kGyIcL/gSQpFeUAx8iWLBqNqalFdO/Aehg4rXvXgjeZiHEa4BhQzwOuGP/96oPj9gimXp26cYcRwAoYIWxfCAGKAYsCyPSpct9lPHlaW3FX6UZ1Vti3YHCuI7oeANisJAKvqvoPBXUPgXDwqf8fff7u7LOrU4bd/u8Ks6vD2dvh1P+v8fvL6spuzdKKacULTAr/Hj9/Amf/mH/xv+YWo3/UPLXU7bTG2vWEfeRKKQcWEHu9b3t9z6CsqvnV++m25VyYY8pF9k3HVbGMq8oAedyuZVY4UfhVVmxq/KVxR5yHfKAr36wWurGkikS2fieu6r2c3F7WQwcneRdqYLpPHItQcjZ+JNXdudTb239iCrUfUk0sHpHN8Z2RdDR+RQg77tDsajaa++6P50jvf6TV2dTV5Beuy6tjOc4fhS0tiBLnCK6Rz6xp68dtzboX1ZusleDbplOke8Hdx6wuoOLFNM16CH9oUzbAPPoOV0Dl57e7wdDi7turhAHKWd6V00177eW6iG5trXOcbU9krltWDUpyuMT6lk5wDmOHWPs7vMrkPKUMpqQrvAqmJDy4uoMrV3bz9NruAJ0MMK9CvVVOFok7WDaKOr7dGJkicdFe6OkX0KV3H+6Xr98Y09GGVE1MoQTFMrtWxohRUafkTSACWJsVUVAWbVLNTQNjsEt4pSttio+4fV8ClXlbILpg0iofeqkrluSzsmCY4Zi4CiOAD9DjH8/BmcnZ4oirxu6HMwScJN9HtKsXm/DIrHDaN4j25cQbLrB1oxEJSy1/AXwIorUYATHAc49kPMincV6RPV4tas0zmYS7LGspCNaRbZcSTfOWtDC0rSJMBRMVZexmqvFMo8zN3kfX7N2GFBKW5Uc1DLB9MHhBY45qa2KeS8VZt73LvkFIBywU/TLDTzWZoFYLUJFATyGoYpL9spfKQzTihW/KRluUtMeTgXKQVmRoLXKqhmyqgp2xNMtkQUmz7S/RqYiX5nNbTMnLK6ur4DmrPlk+Qxb22S3VUE1iSNOciCpTJ1xxPHUmWZn7HIlNybRaeGqqxXQUhBT0CTkO37aKhCxFYBum0MH0KsybuOWA3nu1hgLs2jSimgrAbXitQ5dHknZlmeKa7Ce4nnLuxu+5fyypWJdt/rDyaWalJCuClS0rpU6jB71VOx6VrZog5rVqWmHTbQPQojdBfhqlUsa1rhaKGU8WFwY187IkpY2nH+CJF33ej6PIw4phbFc0xFvmwdFZR6Px+BfBJdIw7qZmO4aLGVyXvZJXKS+5XfQdX2V1P3vUnKssbDnFeOFiA2hgNADGJSMZ/30eW5/v4+6rGv8heH1qCYhxTf4yCrdcgmMXEWm1G2JN9rhtdyvEjUZA9eY5GyZU/xEYe//90ZX8HLl/mLjy68K2HYeFxHSp609uDo7EhhmN6HPmY9BUpavczzCUH0qsF2djJ/mkQkxwJYk0CiZZcPHOTD+QDrtWcbBZDcmJeQKPRLiiA2F4RZsktiPX8LrF7dRSrdg/PsHT4hlLMCVQf12bOz3nmn08liwj2J0jWuzTddSW/nJyUVYHmgevJvpu74PqQkFge8IllvkPPsy0tnOhVf/cFUJKV9a46qUl8NWngT+9qxZGw8OO3Z/z2bOILseDZys8Rik6cn2y/EkneTTXZJ+SxOma5Zu9nbZxAnjj28sfKb1D6wfKZqFLD3pr3SBd3a7itLit00vhQnW1+dFCdkglG0VvexXrnurWf3+xOrY8h/pbkcgCsSnbwa0rIvCeYOp/IGeCBQ1UPGJyiIXGtwNbi0XecTlMqopTjjK0XJ3St7QHQNYRzycoQjf5Uf1spv6H6bj9jLc6qOYMYRT5kSWh0laxrLWk0iDtrhWxu8f5FXpC4H/SoGJKx4YlQLhrziw+j+alSu3oduXlrKSNfiSMwS7HNxacyS6d8+gGqI2+2Ui5RjKvkVlD4iusgbSETQEGmVllEDywKVpnEcxgsV3udBA5pl8FIEDFBe6k4TAx4wPIRRlHUryM7KJfZXMp3KRFWRyZvYOqx8AIWWwlNeqI8wh9AK/+2so2Q1+6/gucr/2vq9lWplubH9GJLtME1AZPH5GlU2qu3friUznKa4JeLWs+WdzlrYqzaqanOwrB3mE4fL/HnRTi4gX7mzttklYbz37NmZbDJorCUTor2FvuoRvrUVuq7ERizuS+ZkH7WYu5qMb74kvE5mo/y9r1RUdtLFpOtM3Rwhux82Nm1yzLhsR1br4skalHUur70N/K24aFR8gvFt4qg3SdfFURz7lC1b1v62tWqN4y1L0XX43VZqtG23biuKvuO2yub1Yqny6lbFNIZ5mshStQGTrGU8JnH1/9VkSbahKiwNSMEne2QcryGhaYxBR0LlPyn/GwAA///Icto70TMAAA=='))]", + "linuxConfiguration": { + "disablePasswordAuthentication": true, + "ssh": { + "publicKeys": [ + { + "path": "[concat('/home/', parameters('adminUsername'), '/.ssh/authorized_keys')]", + "keyData": "[parameters('adminPublicKey')]" + } + ] + } + }, + "secrets": [ + { + "sourceVault": { + "id": "[parameters('pfxKeyVaultResourceId')]" + }, + "vaultCertificates": [ + { + "certificateUrl": "[parameters('pfxKeyVaultSecretUrl')]" + } + ] + } + ] + }, + "hardwareProfile": { + "vmSize": "[parameters('virtualMachineSize')]" + }, + "storageProfile": { + "imageReference": { + "publisher": "[parameters('virtualMachinePublisher')]", + "offer": "[parameters('virtualMachineOffer')]", + "sku": "[parameters('virtualMachineSku')]", + "version": "[parameters('virtualMachineVersion')]" + }, + "osDisk": { + "createOption": "FromImage", + "diskSizeGB": 30, + "managedDisk": { + "storageAccountType": "Premium_LRS" + } + }, + "dataDisks": [ + ] + }, + "networkProfile": { + "networkInterfaces": [ + { + "id": "[resourceId('Microsoft.Network/networkInterfaces',variables('nicName'))]" + } + ] + } + }, + "dependsOn": [ + "[concat('Microsoft.Network/networkInterfaces/',variables('nicName'))]" + ] + }, + { + "apiVersion": "2017-03-30", + "dependsOn": [ + "[concat('Microsoft.Compute/virtualMachines/',variables('rgname'),'-vm')]" + ], + "location": "[resourceGroup().location]", + "name": "[concat(variables('rgname'),'-vm/cse')]", + "properties": { + "protectedSettings": { + "commandToExecute": "[concat(variables('provisionScriptParameters'),' LOCATION=',variables('location'),' TENANT_ID=',variables('tenantId'),' PIP_FQDN=', '\"', reference(resourceId('Microsoft.Network/publicIPAddresses',parameters('pipName')),'2017-10-01').dnsSettings.fqdn,'\"',' /opt/azure/containers/script.sh >> /var/log/azure/docker-registry.log 2>&1')]" + }, + "publisher": "Microsoft.Azure.Extensions", + "settings": { + }, + "type": "CustomScript", + "typeHandlerVersion": "2.0", + "autoUpgradeMinorVersion": true + }, + "type": "Microsoft.Compute/virtualMachines/extensions" + }, + { + "type": "Microsoft.Network/virtualNetworks", + "name": "[concat(variables('rgname'),'-vnet')]", + "apiVersion": "2017-10-01", + "location": "[resourceGroup().location]", + "properties": { + "addressSpace": { + "addressPrefixes": [ + "172.16.0.0/24" + ] + }, + "subnets": [ + { + "name": "default", + "properties": { + "addressPrefix": "172.16.0.0/24" + } + } + ] + } + }, + { + "type": "Microsoft.Network/networkInterfaces", + "name": "[variables('nicName')]", + "apiVersion": "2017-10-01", + "location": "[resourceGroup().location]", + "properties": { + "ipConfigurations": [ + { + "name": "ipconfig1", + "properties": { + "subnet": { + "id": "[variables('subnetRef')]" + }, + "privateIPAllocationMethod": "Dynamic", + "publicIpAddress": { + "id": "[resourceId('Microsoft.Network/publicIpAddresses',parameters('pipName'))]" + } + } + } + ], + "networkSecurityGroup": { + "id": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]" + } + }, + "dependsOn": [ + "[concat('Microsoft.Network/virtualNetworks/',variables('rgname'),'-vnet')]", + "[concat('Microsoft.Network/publicIpAddresses/',parameters('pipName'))]", + "[concat('Microsoft.Network/networkSecurityGroups/',variables('nsgName'))]" + ] + }, + { + "type": "Microsoft.Network/publicIPAddresses", + "sku": { + "name": "Basic" + }, + "name": "[parameters('pipName')]", + "apiVersion": "2017-10-01", + "location": "[resourceGroup().location]", + "properties": { + "publicIpAllocationMethod": "[parameters('pipAllocationMethod')]", + "dnsSettings": { + "domainNameLabel": "[parameters('pipDomainNameLabel')]" + } + } + }, + { + "type": "Microsoft.Network/networkSecurityGroups", + "name": "[variables('nsgName')]", + "apiVersion": "2017-10-01", + "location": "[resourceGroup().location]", + "properties": { + "securityRules": [ + { + "name": "HTTPS", + "properties": { + "priority": 320, + "protocol": "Tcp", + "access": "Allow", + "direction": "Inbound", + "sourceApplicationSecurityGroups": [ + ], + "destinationApplicationSecurityGroups": [ + ], + "sourceAddressPrefix": "*", + "sourcePortRange": "*", + "destinationAddressPrefix": "*", + "destinationPortRange": "443" + } + } + ] + } + } + ], + "outputs": { + "adminUsername": { + "type": "string", + "value": "[parameters('adminUsername')]" + }, + "virtualMachineSku": { + "type": "string", + "value": "[parameters('virtualMachineSku')]" + } + } +} \ No newline at end of file diff --git a/registry/DeploymentTemplates/azuredeploy.parameters-example.json b/registry/DeploymentTemplates/azuredeploy.parameters-example.json new file mode 100644 index 0000000..c424b81 --- /dev/null +++ b/registry/DeploymentTemplates/azuredeploy.parameters-example.json @@ -0,0 +1,60 @@ +{ + "schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "adminUsername": { + "value": "azureuser" + }, + "adminPublicKey": { + "value": "ssh-rsa ...." + }, + "virtualMachineSize": { + "value": "Standard_F8s_v2" + }, + "virtualMachinePublisher": { + "value": "microsoft-aks" + }, + "virtualMachineOffer": { + "value": "aks" + }, + "virtualMachineSku": { + "value": "aks-ubuntu-1604-201910" + }, + "virtualMachineVersion": { + "value": "latest" + }, + "pipName": { + "value": "registry" + }, + "pipDomainNameLabel": { + "value": "registry" + }, + "pipAllocationMethod": { + "value": "dynamic" + }, + "storageAccountResourceId": { + "value": "/subscriptions/{SUBSCRIPTION}/resourceGroups/{RESOURCE-GROUP}/providers/Microsoft.Storage/storageaccounts/{SA-NAME}" + }, + "storageAccountContainer": { + "value": "images" + }, + "pfxKeyVaultResourceId": { + "value": "/subscriptions/{SUBSCRIPTION}/resourceGroups/{RESOURCE-GROUP}/providers/Microsoft.KeyVault/vaults/{KV-NAME}" + }, + "pfxKeyVaultSecretUrl": { + "value": "https://{KV-NAME}.vault.{FQDN}/secrets/{RESOURCE-GROUP}/{SECRET-VERSION}" + }, + "pfxThumbprint": { + "value": "" + }, + "registryReplicas": { + "value": "20" + }, + "servicePrincipalClientId": { + "value": "" + }, + "servicePrincipalClientSecret": { + "value": "" + } + } +} \ No newline at end of file diff --git a/registry/DeploymentTemplates/azuredeploy.parameters.json b/registry/DeploymentTemplates/azuredeploy.parameters.json new file mode 100644 index 0000000..d7e52af --- /dev/null +++ b/registry/DeploymentTemplates/azuredeploy.parameters.json @@ -0,0 +1,60 @@ +{ + "schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentParameters.json#", + "contentVersion": "1.0.0.0", + "parameters": { + "adminUsername": { + "value": "" + }, + "adminPublicKey": { + "value": "" + }, + "virtualMachineSize": { + "value": "" + }, + "virtualMachinePublisher": { + "value": "" + }, + "virtualMachineOffer": { + "value": "" + }, + "virtualMachineSku": { + "value": "" + }, + "virtualMachineVersion": { + "value": "" + }, + "pipName": { + "value": "" + }, + "pipDomainNameLabel": { + "value": "" + }, + "pipAllocationMethod": { + "value": "" + }, + "storageAccountResourceId": { + "value": "" + }, + "storageAccountContainer": { + "value": "" + }, + "pfxKeyVaultResourceId": { + "value": "" + }, + "pfxKeyVaultSecretUrl": { + "value": "" + }, + "pfxThumbprint": { + "value": "" + }, + "registryReplicas": { + "value": "20" + }, + "servicePrincipalClientId": { + "value": "" + }, + "servicePrincipalClientSecret": { + "value": "" + } + } +} \ No newline at end of file diff --git a/registry/DeploymentTemplates/createUiDefinition.json b/registry/DeploymentTemplates/createUiDefinition.json new file mode 100644 index 0000000..e9e06e7 --- /dev/null +++ b/registry/DeploymentTemplates/createUiDefinition.json @@ -0,0 +1,185 @@ +{ + "handler": "Microsoft.Compute.MultiVm", + "version": "0.4.0-preview", + "parameters": { + "basics": [], + "steps": [ + { + "name": "VirtualMachineStep", + "label": "Virtual Machine", + "subLabel": { + "preValidation": "Provide virtual machine settings", + "postValidation": "Done" + }, + "bladeTitle": "Virtual machine configuration", + "elements": [ + { + "name": "adminUsername", + "type": "Microsoft.Compute.UserNameTextBox", + "label": "Username", + "defaultValue": "azureuser", + "constraints": { + "required": true, + "regex": "^[a-z0-9A-Z]{1,30}$", + "validationMessage": "Only alphanumeric characters are allowed, and the value must be 1-30 characters long." + }, + "osPlatform": "Linux" + }, + { + "name": "adminPublicKey", + "type": "Microsoft.Compute.CredentialsCombo", + "label": { + "authenticationType": "Authentication type", + "password": "Password", + "confirmPassword": "Confirm password", + "sshPublicKey": "SSH public key" + }, + "constraints": { + "required": true, + "customPasswordRegex": "^(?=.*[A-Za-z])(?=.*\\d)[A-Za-z\\d]{12,}$", + "customValidationMessage": "The password must be alphanumeric, contain at least 12 characters, and have at least 1 letter and 1 number." + }, + "options": { + "hideConfirmation": false, + "hidePassword": true + }, + "osPlatform": "Linux" + }, + { + "name": "virtualMachineSize", + "type": "Microsoft.Compute.SizeSelector", + "label": "Size", + "osPlatform": "Linux", + "defaultValue": "Standard_F8s_v2", + "recommendedSizes": [ + "Standard_F8s_v2" + ], + "options": { + "hideDiskTypeFilter": false + } + }, + { + "name": "dnsPrefix", + "type": "Microsoft.Common.TextBox", + "label": "Domain name label", + "defaultValue": "registry", + "toolTip": "The domain name is invalid. It can contain only lowercase letters, numbers and hyphens. The first character must be a letter. The last character must be letter or number. The value must be between 3 and 16 characters long.", + "constraints": { + "required": true, + "regex": "^[a-z][a-z0-9-]{1,16}[a-z0-9]$", + "validationMessage": "DNS prefix is invalid. The prefix must conform to the following regular expression: ^[a-z][a-z0-9-]{1,16}[a-z0-9]$." + } + }, + { + "name": "registryReplicas", + "type": "Microsoft.Common.TextBox", + "label": "Replicas", + "defaultValue": "20", + "toolTip": "Registry replica count", + "constraints": { + "required": true, + "regex": "^[0-9]{1,2}$" + } + }, + { + "name": "virtualMachineSku", + "type": "Microsoft.Common.TextBox", + "label": "Image SKU", + "defaultValue": "aks-ubuntu-1604-201910", + "toolTip": "OS image SKU", + "constraints": { + "required": true + } + }, + { + "name": "servicePrincipalClientId", + "type": "Microsoft.Common.TextBox", + "label": "Service Principal Client ID", + "constraints": { + "required": true + } + }, + { + "name": "servicePrincipalClientSecret", + "type": "Microsoft.Common.PasswordBox", + "label": { + "password": "Service Principal password", + "confirmPassword": "Confirm Service Principal password" + }, + "constraints": { + "required": true + } + } + ] + }, + { + "name": "StoragePfxStep", + "label": "Storage and Key Vault", + "subLabel": { + "preValidation": "Provide storage and Key Vault settings", + "postValidation": "Done" + }, + "bladeTitle": "Storage and Key Vault configuration", + "elements": [ + { + "name": "storageAccountResourceId", + "type": "Microsoft.Common.TextBox", + "label": "Existing backend Storage Account resource ID", + "constraints": { + "required": true + } + }, + { + "name": "storageAccountContainer", + "type": "Microsoft.Common.TextBox", + "label": "Existing backend blob container", + "constraints": { + "required": true + } + }, + { + "name": "keyVaultResourceId", + "type": "Microsoft.Common.TextBox", + "label": "PFX Certificate Key Vault Resource Id", + "constraints": { + "required": true + } + }, + { + "name": "keyVaultSecretUrl", + "type": "Microsoft.Common.TextBox", + "label": "PFX Certificate Key Vault Secret URL", + "constraints": { + "required": true + } + }, + { + "name": "certificateThumbprint", + "type": "Microsoft.Common.TextBox", + "label": "PFX Certificate Thumbprint", + "constraints": { + "required": true + } + } + ] + } + ], + "outputs": { + "adminUsername": "[steps('VirtualMachineStep').adminUsername]", + "adminPublicKey": "[steps('VirtualMachineStep').adminPublicKey.sshPublicKey]", + "virtualMachineSize": "[steps('VirtualMachineStep').virtualMachineSize]", + "pipName": "[steps('VirtualMachineStep').dnsPrefix]", + "pipDomainNameLabel": "[steps('VirtualMachineStep').dnsPrefix]", + "pipAllocationMethod": "Static", + "storageAccountResourceId": "[steps('StoragePfxStep').storageAccountResourceId]", + "storageAccountContainer": "[steps('StoragePfxStep').storageAccountContainer]", + "pfxKeyVaultResourceId": "[steps('StoragePfxStep').keyVaultResourceId]", + "pfxKeyVaultSecretUrl": "[steps('StoragePfxStep').keyVaultSecretUrl]", + "pfxThumbprint": "[steps('StoragePfxStep').certificateThumbprint]", + "registryReplicas": "[steps('VirtualMachineStep').registryReplicas]", + "virtualMachineSku": "[steps('VirtualMachineStep').virtualMachineSku]", + "servicePrincipalClientId": "[steps('VirtualMachineStep').servicePrincipalClientId]", + "servicePrincipalClientSecret": "[steps('VirtualMachineStep').servicePrincipalClientSecret]" + } + } +} \ No newline at end of file diff --git a/registry/Icons/Large.png b/registry/Icons/Large.png new file mode 100644 index 0000000..2d274d5 Binary files /dev/null and b/registry/Icons/Large.png differ diff --git a/registry/Icons/Medium.png b/registry/Icons/Medium.png new file mode 100644 index 0000000..fa0fd61 Binary files /dev/null and b/registry/Icons/Medium.png differ diff --git a/registry/Icons/Small.png b/registry/Icons/Small.png new file mode 100644 index 0000000..2afcff0 Binary files /dev/null and b/registry/Icons/Small.png differ diff --git a/registry/Icons/Wide.png b/registry/Icons/Wide.png new file mode 100644 index 0000000..44f2873 Binary files /dev/null and b/registry/Icons/Wide.png differ diff --git a/registry/README.md b/registry/README.md new file mode 100644 index 0000000..9cbd5bb --- /dev/null +++ b/registry/README.md @@ -0,0 +1,55 @@ +# Container Registry Marketplace Item + +## Install Gallery Item + +### Prepare Package + +Use `make release` to create an Azure Gallery Package (.azpkg) for the Container Registry solution template. The package will be placed in the `_out` directory. Upload the package to a BLOB container in an Azure Storage Account with anonymous access enabled. + +### Side-load Package + +Update the admin credentials and storage account information in `gallery.ps1`, and run it to connect to the management endpoint and install the package in your Azure Stack instance. + +```powershell +Scripts\gallery.ps1 +``` + +## Solution Pre-requisites + +### Create Self-signed Certificate (optional) + +Update the certificate information in `self-signed.ps1`, and run it to create a new `pfx` self-signed certificate. If you already have a certificate, you can skip this step. + +```powershell +Scripts\self-signed.ps1 +``` + +This certificate will be uploaded to a Key Vault instance in a subsequent step, and then consumed by the private container registry. + +### Create Backend Storage Account and Key Vault + +Update the tenant information in `connect.ps1` and run it to connect to the tenant space. + +```powershell +Scripts\connect.ps1 +``` + +Set your environment information in `pre-reqs.ps1` and run it to create and populate the required backend Storage Account and Key Vault instance. Take note of the script output as it will be used later to deploy the private container registry. + +```powershell +Scripts\pre-reqs.ps1 +``` + +## Deploy Solution + +From the tenant portal, go to: + +``` +Create a resource -> Compute -> Container Registry +``` + +## Logs + +The container registry logs are located in directory `/var/lib/docker/containers/` + +To query all logs: `cat /var/lib/docker/containers/*/*-json.log | grep "authentication failure"` diff --git a/registry/Screenshots/screenshot.png b/registry/Screenshots/screenshot.png new file mode 100644 index 0000000..a217442 Binary files /dev/null and b/registry/Screenshots/screenshot.png differ diff --git a/registry/Scripts/connect.ps1 b/registry/Scripts/connect.ps1 new file mode 100644 index 0000000..e4c1b71 --- /dev/null +++ b/registry/Scripts/connect.ps1 @@ -0,0 +1,14 @@ +# External FQDN of the instance +$FQDN="" + +$TENANT_NAME = "" +$SUBSCRIPTION = "" + +# Connect to tenant space +Add-AzureRMEnvironment -Name "AzureStackTenant" -ArmEndpoint "https://management.$FQDN" + +$AuthEndpoint = (Get-AzureRmEnvironment -Name "AzureStackTenant").ActiveDirectoryAuthority.TrimEnd('/') +$TenantId = (invoke-restmethod "$($AuthEndpoint)/$($TENANT_NAME)/.well-known/openid-configuration").issuer.TrimEnd('/').Split('/')[-1] +Add-AzureRmAccount -EnvironmentName "AzureStackTenant" -TenantId $TenantId + +Select-AzureRmSubscription -Subscription $SUBSCRIPTION | out-null diff --git a/registry/Scripts/gallery.ps1 b/registry/Scripts/gallery.ps1 new file mode 100644 index 0000000..b8543fe --- /dev/null +++ b/registry/Scripts/gallery.ps1 @@ -0,0 +1,75 @@ +#requires -runasadministrator + +<# +.Synopsis + The script provide functionality to install a marketplace item from given URL. + +.Description + The script provide functionality to install a marketplace item from given URL. + +.Parameter AzureStackDomainName + Fully qualified domain name of Azure Stack instance. + +.Parameter AdminUserName + Admin username. + +.Parameter AdminPassword + Admin password. + +.Parameter TenantId + Tenant ID of Azure Stack. + +.Parameter MarketplaceUri + Marketplace package download URL. + +.Parameter EnvironmentName + Name of the environment. + +.Example + gallery.ps1 -AzureStackDomainName local.microsoft.com + -AdminUserName admin@localazurestack.onmicrosoft.com + -AdminPassword + -TenantId 00000000-0000-0000-0000-000000000000 + -MarketplaceUri https://local.blob.core.windows.net/marketplaceblob/Microsoft.AzureStackContainerRegistry.1.0.0.azpkg +#> +Param +( + [Parameter(Mandatory = $true, HelpMessage = "Fully qualified domain name of Azure Stack instance.")] + [string] $AzureStackDomainName, + [Parameter(Mandatory = $true, HelpMessage = "Admin username.")] + [string] $AdminUserName, + [Parameter(Mandatory = $true, HelpMessage = "Admin password")] + [string] $AdminPassword, + [Parameter(Mandatory = $true, HelpMessage = "Tenant ID of Azure Stack.")] + [string] $TenantId, + [Parameter(Mandatory = $true, HelpMessage = "Marketplace package download URL.")] + [string] $MarketplaceUri, + [Parameter(Mandatory = $false, HelpMessage = "Name of the environment.")] + [string] $EnvironmentName = "AzureStackAdmin" +) + +$environment = Get-AzureRmEnvironment -Name $EnvironmentName +if ($null -eq $environment) +{ + $armEndPoint="https://adminmanagement.$AzureStackDomainName/" + $environment = Add-AzureRmEnvironment -Name $EnvironmentName -ARMEndpoint $armEndPoint -ErrorAction Stop +} +else +{ + Write-Host "Environment($EnvironmentName) is already in use." +} + +$secpasswd = ConvertTo-SecureString $AdminPassword -AsPlainText -Force +$mycreds = New-Object System.Management.Automation.PSCredential ($AdminUserName, $secpasswd) +Login-AzureRmAccount -Credential $mycreds -EnvironmentName $EnvironmentName -TenantId $TenantId + +# Deploy package +#$PKG_SA_NAME="" +#$PKG_SA_CONTAINER="" +#$PKG_VERSION="" +#"https://$PKG_SA_NAME.blob.$AzureStackDomainName/$PKG_SA_CONTAINER/Microsoft.AzureStackContainerRegistry.$PKG_VERSION.azpkg" + +Add-AzsGalleryItem -Force -GalleryItemUri $MarketplaceUri + +# Remove package +# Remove-AzsGalleryItem -Force -Name "Microsoft.AzureStackContainerRegistry.$PKG_VERSION" diff --git a/registry/Scripts/pre-reqs.ps1 b/registry/Scripts/pre-reqs.ps1 new file mode 100644 index 0000000..9ec01b4 --- /dev/null +++ b/registry/Scripts/pre-reqs.ps1 @@ -0,0 +1,664 @@ + + +####################################################################################################################### +####################################################################################################################### +<# +.Synopsis + Check if current user is logged in. + +.Description + Check if current user is logged in. + Throws exption if user is not logged in. + +.Example + Get-AzureStackLoginStatus +#> +function Get-AzureStackLoginStatus () { + $ErrorActionPreference = "SilentlyContinue"; + $subscriptionDetails = Get-AzureRmSubscription -ErrorVariable isTenantLoggedIn + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $subscriptionDetails) { + throw "Login to ARM endpoint is not done. Please login, and select a given subscription." + } +} + +<# +.Synopsis + Create a resource group under selected subscription + +.Description + Create a resource group under selected subscription + It will skip creation of resource group if already present. + +.Parameter ResourceGroupName + Name of the resource group to be created. + +.Example + New-ResourceGroup -ResourceGroupName "resourcegroupname" +#> +function New-ResourceGroup ( + [Parameter(Mandatory = $true, HelpMessage = "Name of the resource group to be created.")] + [string] $ResourceGroupName +) { + Write-Host "Check if resource group ($ResourceGroupName) already exists" + $ErrorActionPreference = "SilentlyContinue"; + $resourceGroupDetails = Get-AzureRmResourceGroup -Name $ResourceGroupName ` + -ErrorVariable resourceGroupExistError + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $resourceGroupDetails) { + # Create resource group + Write-Host "Resource group ($ResourceGroupName) does not exist. Creating a new resource group." + New-AzureRmResourceGroup -Name $ResourceGroupName ` + -Location $Location | Out-Null + $ErrorActionPreference = "SilentlyContinue"; + $newResourceGroupDetails = Get-AzureRmResourceGroup -Name $ResourceGroupName ` + -ErrorVariable resourceGroupExistError + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $newResourceGroupDetails) { + throw "Creation of resource group ($ResourceGroupName) failed." + } + + Write-Host "Resource group ($ResourceGroupName) created successfully." + } + else { + Write-Host "Resource group ($ResourceGroupName) already exists. Skipping creation." + } +} + +<# +.Synopsis + Create a storage account under given resource group + +.Description + Create a storage account under given resource group + It will skip creation of storage account resource if already present. + Return storage account details. + +.Parameter ResourceGroupName + The storage account's resource group. + +.Parameter Location + Location of Azure Stack. + +.Parameter StorageAccountName + Storage account name which needs to be created. + +.Parameter SkuName + Storage account sku name to be used. + +.Parameter EnableHttpsTrafficOnly + Enable flag to have https traffic only. + +.Example + New-StorageAccount -ResourceGroupName "resourcegroupname" + -Location "local" + -StorageAccountName "storageaccountname" + -SkuName "Premium_LRS" + -EnableHttpsTrafficOnly 1 +#> +function New-StorageAccount ( + [Parameter(Mandatory = $true, HelpMessage = "The storage account's resource group.")] + [string] $ResourceGroupName, + [Parameter(Mandatory = $true, HelpMessage = "Location of Azure Stack.")] + [string] $Location, + [Parameter(Mandatory = $true, HelpMessage = "Storage account name which needs to be created.")] + [string] $StorageAccountName, + [Parameter(Mandatory = $false, HelpMessage = "Storage account sku name to be used.")] + [string] $SkuName = "Premium_LRS", + [Parameter(Mandatory = $false, HelpMessage = "Enable flag to have https traffic only.")] + [int] $EnableHttpsTrafficOnly = 1 +) { + # Create storage account + Write-Host "Check if storage account ($StorageAccountName) already exists" + $ErrorActionPreference = "SilentlyContinue"; + $storageAccountDetails = Get-AzureRmStorageAccount -ResourceGroupName $ResourceGroupName ` + -Name $StorageAccountName ` + -ErrorVariable storageAccountExistError + $ErrorActionPreference = "Continue"; #Turning errors back on + if ($storageAccountExistError) { + Write-Host "Storage account does not exist. Creating a new storage account ($StorageAccountName) under resource group ($ResourceGroupName)." + $ErrorActionPreference = "SilentlyContinue"; + New-AzureRmStorageAccount -ResourceGroupName $ResourceGroupName ` + -AccountName $StorageAccountName ` + -Location $Location ` + -SkuName $SkuName ` + -EnableHttpsTrafficOnly $EnableHttpsTrafficOnly | Out-Null + + $newstorageAccountDetails = Get-AzureRmStorageAccount -ResourceGroupName $ResourceGroupName ` + -Name $StorageAccountName ` + -ErrorVariable storageAccountExistError + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $newstorageAccountDetails) { + throw "Creation a new storage account ($StorageAccountName) under resource group ($ResourceGroupName) failed. Check if specified storage account name already exists." + } + $storageAccountDetails = $newstorageAccountDetails + Write-Host "Storage account ($StorageAccountName) created successfully." + } + else { + Write-Host "Storage account ($StorageAccountName) already exists. Skipping creation." + } + + return $storageAccountDetails +} + +<# +.Synopsis + Create a account blob container under given storage account + +.Description + Create a account blob container under given storage account + It will skip creation of storage account blob container if already present. + +.Parameter ResourceGroupName + The storage account's resource group + +.Parameter StorageAccountName + Storage account name under which storage blob needs to be created. + +.Parameter StorageAccountBlobContainer + Storage account blob container name which needs to be created. + +.Example + New-StorageAccountContainer -ResourceGroupName "resourcegroupname" + -StorageAccountName "storageaccountname" + -StorageAccountBlobContainer "images" +#> +function New-StorageAccountContainer ( + [Parameter(Mandatory = $true, HelpMessage = "The storage account's resource group.")] + [string] $ResourceGroupName, + [Parameter(Mandatory = $true, HelpMessage = "Storage account name under which storage blob needs to be created.")] + [string] $StorageAccountName, + [Parameter(Mandatory = $true, HelpMessage = "Storage account blob container name which needs to be created.")] + [string] $StorageAccountBlobContainer +) { + # Storage Account Container + # ============================================= + Write-Host "Check if storage account blob container ($StorageAccountBlobContainer) already exists." + $ErrorActionPreference = "SilentlyContinue"; + $storageContainerDetails = Get-AzureStorageContainer -Name $StorageAccountBlobContainer ` + -ErrorVariable storageContainerExistError + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $storageContainerDetails) { + # Create container under storage account + Write-Host "Blob container does not exist. Creating blob container ($StorageAccountBlobContainer) under storage account ($StorageAccountName)." + New-AzureStorageContainer -Name $StorageAccountBlobContainer | Out-Null + $ErrorActionPreference = "SilentlyContinue"; + $newstorageContainerDetails = Get-AzureStorageContainer -Name $StorageAccountBlobContainer ` + -ErrorVariable storageContainerExistError + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $newstorageContainerDetails) { + throw "Creation of storage blob container ($StorageAccountBlobContainer) failed." + } + } + else { + Write-Host "Storage blob container ($StorageAccountBlobContainer) already exists. Skipping creation." + } +} + +<# +.Synopsis + Create a key vault account under given resource group + +.Description + Create a key vault account under given resource group + It will skip creation of key vault if already present. + Return key vault details. + +.Parameter ResourceGroupName + Name of the resource group under which key vault to be created. + +.Parameter Location + Location of Azure Stack. + +.Parameter KeyVaultName + Keyvault name which needs to be created. + +.Parameter Sku + Sku to be used to create keyVault. Default is standard. + +.Example + New-KeyVault -ResourceGroupName "resourcegroupname" + -Location "local" + -KeyVaultName "keyvault" +#> +function New-KeyVault ( + [Parameter(Mandatory = $true, HelpMessage = "Name of the resource group under which key vault to be created.")] + [string] $ResourceGroupName, + [Parameter(Mandatory = $true, HelpMessage = "Location of Azure Stack.")] + [string] $Location, + [Parameter(Mandatory = $true, HelpMessage = "Keyvault name which needs to be created.")] + [string] $KeyVaultName, + [Parameter(Mandatory = $false, HelpMessage = "Sku to be used to create keyVault.")] + [string] $Sku = "standard" +) { + Write-Host "Check if key vault($KeyVaultName) exists." + $ErrorActionPreference = "SilentlyContinue"; + $keyVaultDetails = Get-AzureRmKeyVault -Name $KeyVaultName -ResourceGroupName $ResourceGroupName + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $keyVaultDetails) { + Write-Host "Creating key vault ($KeyVaultName) as it does not exist." + $ErrorActionPreference = "SilentlyContinue"; + $keyVaultDetails = New-AzureRmKeyVault -ResourceGroupName $ResourceGroupName ` + -VaultName $KeyVaultName ` + -Location $Location ` + -Sku $Sku ` + -EnabledForDeployment + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $keyVaultDetails) { + throw "Creation of KeyVault ($KeyVaultName) failed." + } + } + else { + Write-Host "Key vault ($KeyVaultName) already exists. Skipping creation." + } + + return $keyVaultDetails +} + +<# +.Synopsis + Create a secret under given key vault. + +.Description + Create a secret under given key vault. + It will throw error if already present. + Force update is allowed when SkipExistCheck value is true. + +.Parameter KeyVaultName + Keyvault name under which secret needs to be created. + +.Parameter SecretName + Name of the secret. + +.Parameter SecretValue + Value which needs to be set for given secret name. + +.Parameter ContentType + Type of the content. + +.Parameter SkipExistCheck + Skip validation check and update if set to true. Default is false. + +.Example + New-KeyVaultSecret -KeyVaultName "keyvault" + -SecretName + -SecretValue + -ContentType + -SkipExistCheck $true + +#> +function New-KeyVaultSecret ( + [Parameter(Mandatory = $true, HelpMessage = "Keyvault name under which secret needs to be created.")] + [string] $KeyVaultName, + [Parameter(Mandatory = $true, HelpMessage = "Name of the secret.")] + [string] $SecretName, + [Parameter(Mandatory = $true, HelpMessage = "Value which needs to be set for given secret name.")] + [string] $SecretValue, + [Parameter(Mandatory = $true, HelpMessage = "Type of the content.")] + [string] $ContentType, + [Parameter(Mandatory = $false, HelpMessage = "Skip validation check and update if set to true.")] + [bool] $SkipExistCheck = $false +) { + Write-Host "Check if key vault secret name ($SecretName) exists." + $ErrorActionPreference = "SilentlyContinue"; + $keyVaultSecretDetails = Get-AzureKeyVaultSecret -VaultName $KeyVaultName -Name $SecretName + $ErrorActionPreference = "Continue"; #Turning errors back on + if ((-not $keyVaultSecretDetails) -or $SkipExistCheck) { + Write-Host "Creating key vault secret name ($SecretName) as it does not exist." + $secureSecret = ConvertTo-SecureString -String $SecretValue -AsPlainText -Force + Set-AzureKeyVaultSecret -VaultName $KeyVaultName ` + -Name $SecretName ` + -SecretValue $secureSecret ` + -ContentType $ContentType | Out-Null + $ErrorActionPreference = "SilentlyContinue"; + $keyVaultSecretDetails = Get-AzureKeyVaultSecret -VaultName $KeyVaultName -Name $SecretName + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $keyVaultSecretDetails) { + throw "Creation of key vault secret ($SecretName) failed. Check if the specified key vault name already exists." + } + } + else { + throw "Key vault with given secret name ($SecretName) already exists." + } +} + +<# +.Synopsis + Read and convert certificate in Json enable format. + +.Description + Read and convert certificate in Json enable format. + Returns json based certificate details which can then be added as secret to Key vault. + +.Parameter CertificateFilePath + Full file path where certificate file is present. + +.Parameter CertificatePassword + Password to read the certificate. + +.Example + Get-CertificateEncoded -CertificateFilePath "c:\certificate\cert.pfx" + -CertificatePassword + +#> +function Get-CertificateEncoded ( + [Parameter(Mandatory = $true, HelpMessage = "Full file path where certificate file is present.")] + [string] $CertificateFilePath, + [Parameter(Mandatory = $true, HelpMessage = "Password to read the certificate.")] + [string] $CertificatePassword +) { + # Store certificate as secret + $fileContentBytes = get-content $CertificateFilePath -Encoding Byte + $fileContentEncoded = [System.Convert]::ToBase64String($fileContentBytes) + $jsonObject = @" + { + "data": "$filecontentencoded", + "dataType" :"pfx", + "password": "$CertificatePassword" + } +"@ + $jsonObjectBytes = [System.Text.Encoding]::UTF8.GetBytes($jsonObject) + $jsonEncoded = [System.Convert]::ToBase64String($jsonObjectBytes) + return $jsonEncoded +} + + +####################################################################################################################### +####################################################################################################################### + +<# +.Synopsis + Create or update existing cert secret under given key vault. + +.Description + Create or update existing cert secret under given key vault. + Force update is allowed when SkipExistCheck value is true. + +.Parameter KeyVaultName + Keyvault name under which secret needs to be created. + +.Parameter CertificateSecretName + Secret name for the certificate which needs to be created in keyvault. + +.Parameter CertificateFilePath + Full file path where certificate file is present. + +.Parameter CertificatePassword + Password to read the certificate. + +.Parameter SkipExistCheck + Skip validation check and update if set to true. Default is false. + +.Example + Set-CertificateSecret -KeyVaultName "keyvault" + -CertificateSecretName + -CertificateFilePath "c:\certificate\cert.pfx" + -CertificatePassword + -SkipExistCheck $true +#> +function Set-CertificateSecret ( + [Parameter(Mandatory = $true, HelpMessage = "Keyvault name which needs to be created.")] + [string] $KeyVaultName, + [Parameter(Mandatory = $true, HelpMessage = "Secret name for the certificate which needs to be created in keyvault.")] + [string] $CertificateSecretName, + [Parameter(Mandatory = $true, HelpMessage = "Full file path where certificate file is present.")] + [string] $CertificateFilePath, + [Parameter(Mandatory = $true, HelpMessage = "Password to read the certificate.")] + [string] $CertificatePassword, + [Parameter(Mandatory = $false, HelpMessage = "Skip validation check and update if set to true.")] + [bool] $SkipExistCheck = $false +) { + # Assuming tenant is logged in already and selected the given subscription. + # best case check if tenant is loged in. Currently subscription selected status is not checked. + Get-AzureStackLoginStatus + # Store certificate as secret + $secret = Get-CertificateEncoded -CertificateFilePath $CertificateFilePath ` + -CertificatePassword $CertificatePassword + # Secret related to certificate. + New-KeyVaultSecret -KeyVaultName $KeyVaultName ` + -SecretName $CertificateSecretName ` + -SecretValue $secret ` + -ContentType pfx ` + -SkipExistCheck $SkipExistCheck + + $keyVaultDetails = Get-AzureRmKeyVault -Name $KeyVaultName + $keyVaultSecretDetails = Get-AzureKeyVaultSecret -VaultName $KeyVaultName ` + -Name $CertificateSecretName + + $mypwd = ConvertTo-SecureString -String $CertificatePassword -Force -AsPlainText + $pfxData = Get-PfxData -FilePath $CertificateFilePath -Password $mypwd + + Write-Host "----------------------------------------------------------------" + Write-Host "PFX KeyVaultResourceId : $($keyVaultDetails.ResourceId)" + Write-Host "PFX KeyVaultSecretUrl : $($keyVaultSecretDetails.Id)" + Write-Host "PFX Certificate Thumbprint : $($pfxData.EndEntityCertificates.Thumbprint)" + Write-Host "----------------------------------------------------------------" +} + +<# +.Synopsis + Create or update existing secret under given key vault. + +.Description + Create or update existing secret under given key vault. + Force update is allowed when SkipExistCheck value is true. + +.Parameter KeyVaultName + Keyvault name under which secret needs to be created. + +.Parameter RegistryUserName + User name using which images will be push and pull. + +.Parameter RegistryUserPassword + Password using which images will be push and pull. + +.Parameter SkipExistCheck + Skip validation check and update if set to true. Default is false. + +.Example + Set-RegistryAccessSecret -KeyVaultName "keyvault" + -RegistryUserName + -RegistryUserPassword + -SkipExistCheck $true +#> +function Set-RegistryAccessSecret ( + [Parameter(Mandatory = $true, HelpMessage = "Keyvault name which needs to be created.")] + [string] $KeyVaultName, + [Parameter(Mandatory = $true, HelpMessage = "User name using which images will be push and pull.")] + [string] $RegistryUserName, + [Parameter(Mandatory = $true, HelpMessage = "Password using which images will be push and pull.")] + [string] $RegistryUserPassword, + [Parameter(Mandatory = $false, HelpMessage = "Skip validation check and update if set to true.")] + [bool] $SkipExistCheck = $false +) { + # Assuming tenant is logged in already and selected the given subscription. + # best case check if tenant is loged in. Currently subscription selected status is not checked. + Get-AzureStackLoginStatus + New-KeyVaultSecret -KeyVaultName $KeyVaultName ` + -SecretName $RegistryUserName ` + -SecretValue $RegistryUserPassword ` + -ContentType "user credentials" ` + -SkipExistCheck $SkipExistCheck +} + +<# +.Synopsis + Create all the pre-requisite required for container registry. + +.Description + Create all the pre-requisite required for container registry. + +.Parameter Location + Location of Azure Stack. + +.Parameter ServicePrincipalId + Service principal ID which will be added to provide contributor access + +.Parameter ResourceGroupName + Name of the resource group to be created. + +.Parameter StorageAccountName + Storage account name which needs to be created. + +.Parameter StorageAccountBlobContainer + Storage account blob container name which needs to be created. + +.Parameter KeyVaultName + Keyvault name which needs to be created. + +.Parameter CertificateSecretName + Secret name for the certificate which needs to be created in keyvault. + +.Parameter CertificateFilePath + Full file path where certificate file is present. + +.Parameter CertificatePassword + Password to read the certificate. + +.Parameter RegistryUserName + User name using which images will be push and pull. + +.Parameter RegistryUserPassword + Password using which images will be push and pull. + +.Example + Set-ContainerRegistryPrerequisites + +#> +function Set-ContainerRegistryPrerequisites +( + [Parameter(Mandatory = $true, HelpMessage = "Location of Azure Stack.")] + [string] $Location, + [Parameter(Mandatory = $true, HelpMessage = "Service principal ID which will be added to provide contributor access.")] + [string] $ServicePrincipalId, + [Parameter(Mandatory = $true, HelpMessage = "Name of the resource group to be created.")] + [string] $ResourceGroupName, + [Parameter(Mandatory = $true, HelpMessage = "Storage account name which needs to be created.")] + [string] $StorageAccountName, + [Parameter(Mandatory = $true, HelpMessage = "Storage account blob container name which needs to be created.")] + [string] $StorageAccountBlobContainer, + [Parameter(Mandatory = $true, HelpMessage = "Keyvault name which needs to be created.")] + [string] $KeyVaultName, + [Parameter(Mandatory = $true, HelpMessage = "Secret name for the certificate which needs to be created in keyvault.")] + [string] $CertificateSecretName, + [Parameter(Mandatory = $true, HelpMessage = "Full file path where certificate file is present.")] + [string] $CertificateFilePath, + [Parameter(Mandatory = $true, HelpMessage = "Password to read the certificate.")] + [string] $CertificatePassword, + [Parameter(Mandatory = $true, HelpMessage = "User name using which images will be push and pull.")] + [string] $RegistryUserName, + [Parameter(Mandatory = $true, HelpMessage = "Password using which images will be push and pull.")] + [string] $RegistryUserPassword +) { + # Assuming tenant is logged in already and selected the given subscription. + # best case check if tenant is loged in. Currently subscription selected status is not checked. + Get-AzureStackLoginStatus + + # Create resource group. In case exists skip creation. + New-ResourceGroup -ResourceGroupName $ResourceGroupName + + # Create storage account. In case exists skip creation. + $storageAccountDetails = New-StorageAccount -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -StorageAccountName $StorageAccountName + + # Create current context to given storage account. + Set-AzureRmCurrentStorageAccount -ResourceGroupName $ResourceGroupName ` + -AccountName $StorageAccountName | out-null + + # Create new storage blob container. + New-StorageAccountContainer -ResourceGroupName $ResourceGroupName ` + -StorageAccountName $StorageAccountName ` + -StorageAccountBlobContainer $StorageAccountBlobContainer + + Write-Host "Checking if ServicePrincipalId ($ServicePrincipalId) already has access on storage account ($StorageAccountName), ." + $ErrorActionPreference = "SilentlyContinue"; + $roleAssignment = Get-AzureRMRoleAssignment -ServicePrincipalName $ServicePrincipalId ` + -Scope $storageAccountDetails.Id ` + -ErrorVariable accessExistError + $ErrorActionPreference = "Continue"; #Turning errors back on + if (-not $roleAssignment) { + Write-Host "Assigning servicePrincipalId ($ServicePrincipalId) contributor role on storage account ($StorageAccountName)" + New-AzureRMRoleAssignment -ApplicationId $ServicePrincipalId ` + -RoleDefinitionName "Contributor" ` + -Scope $storageAccountDetails.Id + } + else { + Write-Host "ServicePrincipalId ($ServicePrincipalId) already has access on Storage account ($StorageAccountName) " + } + + # Create key vault enabled for deployment + New-KeyVault -ResourceGroupName $ResourceGroupName ` + -Location $Location ` + -KeyVaultName $KeyVaultName ` + -Sku standard | Out-Null + Write-Host "Set access policy on keyvault ($KeyVaultName) for client ($ServicePrincipalId)" + Set-AzureRmKeyVaultAccessPolicy -VaultName $KeyVaultName ` + -ServicePrincipalName $ServicePrincipalId ` + -PermissionsToSecrets GET, LIST + + # Secret related to registry credentials. + Set-RegistryAccessSecret -KeyVaultName $KeyVaultName ` + -RegistryUserName $RegistryUserName ` + -RegistryUserPassword $RegistryUserPassword ` + -SkipExistCheck $true + + # Store certificate as secret + Set-CertificateSecret -KeyVaultName $KeyVaultName ` + -CertificateSecretName $CertificateSecretName ` + -CertificateFilePath $CertificateFilePath ` + -CertificatePassword $CertificatePassword ` + -SkipExistCheck $true + + Write-Host "StorageAccountResourceId : $($storageAccountDetails.Id)" + Write-Host "Blob Container : $StorageAccountBlobContainer" + Write-Host "----------------------------------------------------------------" + Get-VMImageSku -Location $Location + Write-Host "----------------------------------------------------------------" +} + +<# +.Synopsis + Returns set of SKUs available for given publisher and offer. + +.Description + Returns set of SKUs available for given publisher and offer. + +.Parameter Location + Location where Azure Stack is deployed. + +.Parameter PublisherName + Publisher name of given image. Default value is microsoft-aks. + +.Parameter Offer + Offer name of given image. Default value is aks. + +.Example + Get-VMImageSku -Location local +#> +function Get-VMImageSku ( + [Parameter(Mandatory = $true, HelpMessage = "Location of Azure Stack.")] + [string] $Location, + [Parameter(Mandatory = $false, HelpMessage = "Publisher name of given image.")] + [string] $PublisherName = "microsoft-aks", + [Parameter(Mandatory = $false, HelpMessage = "Offer name of given image.")] + [string] $Offer = "aks" +) { + # Assuming tenant is logged in already and selected the given subscription. + # best case check if tenant is loged in. Currently subscription selected status is not checked. + Get-AzureStackLoginStatus + + $ErrorActionPreference = "SilentlyContinue"; + $skuDetails = Get-AzureRmVMImageSku -Location $Location -PublisherName $PublisherName -Offer $Offer + $ErrorActionPreference = "Continue"; #Turning errors back on + if ($skuDetails) { + Write-Host "Available Skus:" + $skuDetails | ForEach-Object { + '{0}' -f $_.Skus + } + #$skuDetails | Select-Object Skus + } + else { + Write-Host "No image sku found with publisher ($PublisherName) and offer ($Offer)." + } +} \ No newline at end of file diff --git a/registry/Scripts/script.sh b/registry/Scripts/script.sh new file mode 100644 index 0000000..0228f49 --- /dev/null +++ b/registry/Scripts/script.sh @@ -0,0 +1,358 @@ +#!/bin/bash + +ERR_APT_INSTALL_TIMEOUT=9 # Timeout installing required apt packages +ERR_MISSING_CRT_FILE=10 # Bad cert thumbprint OR pfx not in key vault OR template misconfigured VM secrets section +ERR_MISSING_KEY_FILE=11 # Bad cert thumbprint OR pfx not in key vault OR template misconfigured VM secrets section +ERR_MISSING_USER_CREDENTIALS=12 # No user credentials secret found on given key vault +ERR_REGISTRY_NOT_RUNNING=13 # The container registry failed to start successfully +ERR_MOBY_APT_LIST_TIMEOUT=25 # Timeout waiting for moby apt sources +ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT=26 # Timeout waiting for MS GPG key download +ERR_MOBY_INSTALL_TIMEOUT=27 # Timeout waiting for moby install +ERR_METADATA=30 # Error querying metadata +ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT=42 # Timeout waiting for https://packages.microsoft.com/config/ubuntu/16.04/packages-microsoft-prod.deb +ERR_MS_PROD_DEB_PKG_ADD_FAIL=43 # Failed to add repo pkg file +ERR_REGISTRY_LOGIN_FAILED=50 # Failed to log into the docker registry +ERR_REGISTRY_PUSH_FAILED=51 # Failed to push images to the docker registry +ERR_REGISTRY_PULL_FAILED=52 # Failed to pull images from the docker registry +ERR_REGISTRY_BUILD_FAILED=53 # Failed to build image locally +ERR_REGISTRY_REMOVE_FAILED=54 # Failed to remove image locally +ERR_APT_UPDATE_TIMEOUT=99 # Timeout waiting for apt-get update to complete + +cleanUpGPUDrivers() { + rm -Rf $GPU_DEST + rm -f /etc/apt/sources.list.d/nvidia-docker.list + apt-key del $(apt-key list | grep NVIDIA -B 1 | head -n 1 | cut -d "/" -f 2 | cut -d " " -f 1) +} + +retrycmd_if_failure() { + retries=$1; wait_sleep=$2; timeout=$3; + shift && shift && shift + for i in $(seq 1 $retries); do + timeout $timeout ${@} && break || \ + if [ $i -eq $retries ]; then + return 1 + else + sleep $wait_sleep + fi + done +} +wait_for_apt_locks() { + while fuser /var/lib/dpkg/lock /var/lib/apt/lists/lock /var/cache/apt/archives/lock >/dev/null 2>&1; do + echo 'Waiting for release of apt locks' + sleep 3 + done +} +apt_get_update() { + retries=10 + apt_update_output=/tmp/apt-get-update.out + for i in $(seq 1 $retries); do + wait_for_apt_locks + dpkg --configure -a + apt-get -f -y install + ! (apt-get update 2>&1 | tee $apt_update_output | grep -E "^([WE]:.*)|([eE]rr.*)$") && \ + cat $apt_update_output && break || \ + cat $apt_update_output + if [ $i -eq $retries ]; then + return 1 + else sleep 5 + fi + done + wait_for_apt_locks +} +apt_get_install() { + retries=$1; wait_sleep=$2; timeout=$3; + shift && shift && shift + for i in $(seq 1 $retries); do + wait_for_apt_locks + dpkg --configure -a + apt-get install -o Dpkg::Options::="--force-confold" --no-install-recommends -y ${@} && break || \ + if [ $i -eq $retries ]; then + return 1 + else + sleep $wait_sleep + apt_get_update + fi + done + wait_for_apt_locks +} +installDeps() { + UBUNTU_RELEASE=$(lsb_release -r -s) + MOBY_VERSION="3.0.6" + + retrycmd_if_failure 120 5 25 curl https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/prod.list > /tmp/microsoft-prod.list || exit $ERR_MOBY_APT_LIST_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft-prod.list /etc/apt/sources.list.d/ || exit $ERR_MOBY_APT_LIST_TIMEOUT + retrycmd_if_failure 120 5 25 curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /tmp/microsoft.gpg || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + retrycmd_if_failure 10 5 10 cp /tmp/microsoft.gpg /etc/apt/trusted.gpg.d/ || exit $ERR_MS_GPG_KEY_DOWNLOAD_TIMEOUT + apt_get_update || exit $ERR_APT_UPDATE_TIMEOUT + + apt_get_install 20 30 120 apache2-utils moby-engine=${MOBY_VERSION} moby-cli=${MOBY_VERSION} --allow-downgrades || exit $ERR_MOBY_INSTALL_TIMEOUT + usermod -aG docker ${ADMIN_USER_NAME} + + for apt_package in curl jq; do + if ! apt_get_install 30 1 600 $apt_package; then + journalctl --no-pager -u $apt_package + exit $ERR_APT_INSTALL_TIMEOUT + fi + done +} +fetchOAuth() { + ENDPOINTS=$(mktemp) + curl -s --retry 5 --retry-delay 10 --max-time 60 -f \ + https://management.${FQDN}/metadata/endpoints?api-version=2015-01-01 > ${ENDPOINTS} + + if [ $? -ne 0 ]; then + exit $ERR_METADATA + fi + + OAUTH=$(jq -r .authentication.loginEndpoint ${ENDPOINTS}) + echo ${OAUTH} | grep -e "/adfs$" + + if [ $? -eq 0 ]; then + TOKEN_URL="${OAUTH}/oauth2/token" + else + TOKEN_URL="${OAUTH}${TENANT_ID}/oauth2/token" + fi +} +fetchCredentials() { + RESOURCE=$(jq -r .authentication.audiences[0] ${ENDPOINTS} | sed "s|https://management.|https://vault.|") + + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X POST \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials" \ + -d "client_id=${SPN_CLIENT_ID}" \ + --data-urlencode "client_secret=${SPN_CLIENT_SECRET}" \ + --data-urlencode "resource=${RESOURCE}" \ + ${TOKEN_URL} | jq -r '.access_token') + + KV_URL="https://${KV_NAME}.vault.${FQDN}/secrets" + SECRETS=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f \ + "${KV_URL}?api-version=2016-10-01" -H "Authorization: Bearer ${TOKEN}" | jq -r .value[].id) + + rm .htpasswd + touch .htpasswd + for secret in ${SECRETS} + do + SECRET_NAME_VERSION="${secret//$KV_URL}" + SECRET_NAME=$(echo ${SECRET_NAME_VERSION} | cut -d '/' -f 2) + SECRET_VALUE=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f \ + "${secret}?api-version=2016-10-01" -H "Authorization: Bearer ${TOKEN}" | jq -r .value) + if [ ${#SECRET_VALUE} -le 255 ]; then + htpasswd -Bb .htpasswd ${SECRET_NAME} ${SECRET_VALUE} + else + echo "Skipping ${SECRET_NAME} as secret value length is more than 255" + fi + done + + if [ ! -s .htpasswd ]; then + echo "file .htpasswd is empty, credentials were not created or there was an error fetching credentials from keyvault" + exit $ERR_MISSING_USER_CREDENTIALS + fi +} +fetchValidationCredentials() { + RESOURCE=$(jq -r .authentication.audiences[0] ${ENDPOINTS} | sed "s|https://management.|https://vault.|") + + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X POST \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials" \ + -d "client_id=${SPN_CLIENT_ID}" \ + --data-urlencode "client_secret=${SPN_CLIENT_SECRET}" \ + --data-urlencode "resource=${RESOURCE}" \ + ${TOKEN_URL} | jq -r '.access_token') + + KV_URL="https://${KV_NAME}.vault.${FQDN}/secrets" + SECRETS=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f \ + "${KV_URL}?api-version=2016-10-01" -H "Authorization: Bearer ${TOKEN}" | jq -r .value[].id) + + for secret in ${SECRETS} + do + SECRET_NAME_VERSION="${secret//$KV_URL}" + REGISTRY_USER=$(echo ${SECRET_NAME_VERSION} | cut -d '/' -f 2) + REGISTRY_PASSWORD=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f \ + "${secret}?api-version=2016-10-01" -H "Authorization: Bearer ${TOKEN}" | jq -r .value) + break + done +} +fetchStorageKeys() { + RESOURCE=$(jq -r .authentication.audiences[0] ${ENDPOINTS}) + + TOKEN=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X POST \ + -H "Content-Type: application/x-www-form-urlencoded" \ + -d "grant_type=client_credentials" \ + -d "client_id=${SPN_CLIENT_ID}" \ + --data-urlencode "client_secret=${SPN_CLIENT_SECRET}" \ + --data-urlencode "resource=${RESOURCE}" \ + ${TOKEN_URL} | jq -r '.access_token') + + SA_URL="https://management.${FQDN}/${SA_RESOURCE_ID}/listKeys?api-version=2017-10-01" + SA_KEY=$(curl -s --retry 5 --retry-delay 10 --max-time 60 -f -X POST \ + "${SA_URL}" -H "Authorization: Bearer ${TOKEN}" -H "Content-Length: 0" | jq -r ".keys[0].value") +} + +echo ADMIN_USER_NAME: ${ADMIN_USER_NAME} +echo CERT_THUMBPRINT: ${CERT_THUMBPRINT} +echo CONTAINER_STATUS_WAIT_TIME: ${CONTAINER_STATUS_WAIT_TIME} +echo ENABLE_VALIDATIONS: ${ENABLE_VALIDATIONS} +echo KV_RESOURCE_ID: ${KV_RESOURCE_ID} +echo LOCATION: ${LOCATION} +echo MARKETPLACE_VERSION: ${MARKETPLACE_VERSION} +echo PIP_FQDN: ${PIP_FQDN} +echo PIP_LABEL: ${PIP_LABEL} +echo REGISTRY_REPLICAS: ${REGISTRY_REPLICAS} +echo REGISTRY_TAG: ${REGISTRY_TAG} +echo SA_CONTAINER: ${SA_CONTAINER} +echo SA_RESOURCE_ID: ${SA_RESOURCE_ID} +echo SPN_CLIENT_ID: ${SPN_CLIENT_ID} +echo TENANT_ID: ${TENANT_ID} + +cleanUpGPUDrivers +SA_NAME=$(echo ${SA_RESOURCE_ID} | awk -F"/" '{print $NF}') +KV_NAME=$(echo ${KV_RESOURCE_ID} | awk -F"/" '{print $NF}') + +EXT_DOMAIN_NAME="${PIP_FQDN//$PIP_LABEL.$LOCATION.cloudapp.}" +FQDN=${LOCATION}.${EXT_DOMAIN_NAME} + +CRT_FILE="${CERT_THUMBPRINT}.crt" +KEY_FILE="${CERT_THUMBPRINT}.prv" +SECRET=$(openssl rand -base64 32) + +if [ -f /opt/azure/vhd-install.complete ]; then + echo "aks base image; skipping dependencies installation" + rm -rf /home/packer + deluser packer + groupdel packer +else + installDeps +fi + +echo validating dependencies +if [ ! -f "/var/lib/waagent/${CRT_FILE}" ]; then + exit $ERR_MISSING_CRT_FILE +fi + +if [ ! -f "/var/lib/waagent/${KEY_FILE}" ]; then + exit $ERR_MISSING_KEY_FILE +fi + +echo adding certs to the ca-store +cp "/var/lib/waagent/Certificates.pem" "/usr/local/share/ca-certificates/azsCertificate.crt" +update-ca-certificates + +echo copy user cert to mount point +STORE="/etc/ssl/certs/registry" +mkdir -p $STORE +cp "/var/lib/waagent/${CRT_FILE}" "${STORE}/${CRT_FILE}" +cp "/var/lib/waagent/${KEY_FILE}" "${STORE}/${KEY_FILE}" + +echo getting management endpoints +fetchOAuth + +echo fetching storage key +fetchStorageKeys + +echo fetching user credentials +HTPASSWD_DIR="/root/auth" +mkdir -p $HTPASSWD_DIR +fetchCredentials +cp .htpasswd $HTPASSWD_DIR/.htpasswd + +echo fetching available registry image tag +REGISTRY_IMAGE_TAG=$(docker images --filter=reference='registry:*' --format "{{.Tag}}" | head -n 1) +if [ -z "$REGISTRY_IMAGE_TAG" ]; then + echo using registry tag passed as no image tag found + REGISTRY_IMAGE_TAG=$REGISTRY_TAG +else + echo using registry tag retireved from query +fi +echo REGISTRY_IMAGE_TAG: ${REGISTRY_IMAGE_TAG} + +echo starting registry container +cat <> docker-compose.yml +version: '3' +services: + registry: + image: registry:${REGISTRY_IMAGE_TAG} + deploy: + mode: replicated + replicas: ${REGISTRY_REPLICAS} + restart_policy: + condition: on-failure + delay: 5s + ports: + - "443:5000" + volumes: + - /etc/ssl/certs:/etc/ssl/certs:ro + - /root/auth:/auth + environment: + - REGISTRY_LOG_ACCESSLOG_DISABLED=false + - REGISTRY_STORAGE=azure + - REGISTRY_STORAGE_AZURE_ACCOUNTNAME=${SA_NAME} + - REGISTRY_STORAGE_AZURE_ACCOUNTKEY=${SA_KEY} + - REGISTRY_STORAGE_AZURE_CONTAINER=${SA_CONTAINER} + - REGISTRY_STORAGE_AZURE_REALM=${FQDN} + - REGISTRY_AUTH=htpasswd + - REGISTRY_AUTH_HTPASSWD_PATH=/auth/.htpasswd + - REGISTRY_AUTH_HTPASSWD_REALM="Registry Realm" + - REGISTRY_HTTP_ADDR=0.0.0.0:5000 + - REGISTRY_HTTP_SECRET=${SECRET} + - REGISTRY_HTTP_TLS_KEY=/etc/ssl/certs/registry/${KEY_FILE} + - REGISTRY_HTTP_TLS_CERTIFICATE=/etc/ssl/certs/registry/${CRT_FILE} +EOF + +docker swarm init +docker stack deploy registry -c docker-compose.yml + +echo validating container status +i=0 +while [ $i -lt $CONTAINER_STATUS_WAIT_TIME ];do + CID=$(docker ps | grep "registry_registry.1\." | head -c 12) + STATUS=$(docker inspect ${CID} | jq ".[0].State.Status" | xargs) + if [[ ! $STATUS == "running" ]]; then + echo containers are not up. we will retry to check the status + sleep 30s + else + break + fi + let i=i+30 +done + +if [[ ! $STATUS == "running" ]]; then + exit $ERR_REGISTRY_NOT_RUNNING +else + echo registry containers are up and running +fi + +if [ $ENABLE_VALIDATIONS == "true" ]; then + echo validating docker registry + REGISTRY_USER="" + REGISTRY_PASSWORD="" + fetchValidationCredentials + docker login localhost:443 -u $REGISTRY_USER -p $REGISTRY_PASSWORD + if [ $? -ne 0 ]; then + exit $ERR_REGISTRY_LOGIN_FAILED + fi + cat <> Dockerfile + FROM registry:${REGISTRY_IMAGE_TAG} + RUN touch registry +EOF + + TEST_IMAGE_NAME="localhost:443/testbuild" + docker build -t ${TEST_IMAGE_NAME} -f Dockerfile . + if [ $? -ne 0 ]; then + exit $ERR_REGISTRY_BUILD_FAILED + fi + docker push ${TEST_IMAGE_NAME} + if [ $? -ne 0 ]; then + exit $ERR_REGISTRY_PUSH_FAILED + fi + docker rmi ${TEST_IMAGE_NAME} + if [ $? -ne 0 ]; then + exit $ERR_REGISTRY_REMOVE_FAILED + fi + docker pull ${TEST_IMAGE_NAME} + if [ $? -ne 0 ]; then + exit $ERR_REGISTRY_PULL_FAILED + fi +fi + +echo "Registry setup done. Remove non required images." +sudo docker system prune -a -f & diff --git a/registry/Scripts/self-signed.ps1 b/registry/Scripts/self-signed.ps1 new file mode 100644 index 0000000..c41ce9b --- /dev/null +++ b/registry/Scripts/self-signed.ps1 @@ -0,0 +1,74 @@ +#requires -runasadministrator +<# +.Synopsis + The script provide functionality to create self signed certificate. + +.Description + The script provide functionality to create self signed certificate. + +.Parameter CertificateCN + Certificate common name. + +.Parameter CertificatePassword + Certificate Password. + +.Parameter CertificateFileExportPath + Certificate file export path including certificate filename. + +.Parameter GenerateWildcardCert + Flag to generate wildcard cert. + +.Example + Self-Signed.ps1 -CertificateCN "registry.local.microsoft.com" + -CertificatePassword + -CertificateFileExportPath c:\certificate\registry_cert.pfx +#> +Param +( + [Parameter(Mandatory = $true, HelpMessage = "Certificate common name.")] + [string] $CertificateCN, + [Parameter(Mandatory = $true, HelpMessage = "Certificate Password.")] + [string] $CertificatePassword, + [Parameter(Mandatory = $true, HelpMessage = "Certificate file export path including certificate filename.")] + [string] $CertificateFileExportPath, + [Parameter(Mandatory = $false, HelpMessage = "Flag to generate wildcard cert.")] + [bool] $GenerateWildcardCert = $false +) + +if (-not (Test-Path -Path $CertificateFileExportPath -IsValid)) +{ + throw "Error: CertificateFileExportPath is not a valid path." +} + +if (Test-Path -Path $CertificateFileExportPath) +{ + throw "Error: File($CertificateFileExportPath) already exist. Please remove to provide a different name." +} + +# Create a self-signed certificate +if ($GenerateWildcardCert){ + Write-Host "Generating wildcard cert for $CertificateCN." + $ssc = New-SelfSignedCertificate -Subject *.$CertificateCN -certstorelocation cert:\LocalMachine\My -dnsname $CertificateCN, *.$CertificateCN +} +else { + Write-Host "Generating normal cert for $CertificateCN." + $ssc = New-SelfSignedCertificate -certstorelocation cert:\LocalMachine\My -dnsname $CertificateCN +} + +if ($ssc){ + Write-Host "Certificate created successfully. Now exporting the certificate." +} +else { + throw "Error: Creation of certificate failed." +} + +$crt = "cert:\localMachine\my\" + $ssc.Thumbprint +$pwd = ConvertTo-SecureString -String $CertificatePassword -Force -AsPlainText +Export-PfxCertificate -cert $crt -FilePath $CertificateFileExportPath -Password $pwd -Force | Out-Null +if (Test-Path -Path $CertificateFileExportPath) { + Write-Host "Certificate ($CertificateFileExportPath) exported successfully." +} +else { + throw "Error: Export of certificate failed." +} + diff --git a/registry/Strings/resources.resjson b/registry/Strings/resources.resjson new file mode 100644 index 0000000..05b95b1 --- /dev/null +++ b/registry/Strings/resources.resjson @@ -0,0 +1,8 @@ +{ + "displayName": "Container Registry Template", + "publisherDisplayName": "Microsoft", + "summary": "Deploy and manage a Docker private registry", + "longSummary": "Deploy and manage a Docker private registry", + "description": "

Deploy and manage a Docker private registry

Container Registry Template is a solution template that deploys and configures a Docker private registry for hosting container images in an Ubuntu VM image. Using the Container Registry Template, you can store and distribute Docker-formatted images for all types of container deployments. Users can benefit from using familiar tooling capable of working with the open source Docker Registry v2. This solution template requires the AKS Base Image for deployment.

Use Container Registry Template to: