Skip to content

Commit bde8d6d

Browse files
ciaran28ciaran28
authored
Merge pull request #36 from ciaran28/main
Major Updates + Functionality
2 parents f793376 + 5d4c6ca commit bde8d6d

File tree

36 files changed

+1298
-339
lines changed

36 files changed

+1298
-339
lines changed

‎.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsAzureLogin.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,12 @@ az upgrade
44
echo $ARM_CLIENT_ID
55
echo $ARM_CLIENT_SECRET
66
echo $ARM_TENANT_ID
7+
echo $AuthenticationType
78

89
az config set extension.use_dynamic_install=yes_without_prompt
910

11+
12+
echo "Service Principal Authentication"
1013
az login --service-principal -u $ARM_CLIENT_ID -p $ARM_CLIENT_SECRET --tenant $ARM_TENANT_ID
14+
az account list
1115

12-
az account list

‎.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsCreateSecretScopes.sh‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
1+
az config set extension.use_dynamic_install=yes_without_promp
2+
az extension add --name application-insights
3+
14
echo $RESOURCE_GROUP_NAME
5+
echo $DATABRICKS_INSTANCE
6+
echo $WORKSPACE_ID
27

38
APP_INSIGHT_NAME=$(az resource list \
49
-g $RESOURCE_GROUP_NAME \
@@ -13,9 +18,8 @@ APP_INSIGHT_INSTRUMENT_KEY=$( az monitor app-insights component show \
1318

1419
echo "Test"
1520

16-
echo $ARM_CLIENT_ID
17-
echo $ARM_TENANT_ID
18-
echo $ARM_CLIENT_SECRET
21+
echo $APP_INSIGHT_NAME
22+
echo $APP_INSIGHT_INSTRUMENT_KEY
1923

2024
echo "Creating Secret Scopes...."
2125

‎.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsMIAuthentication.sh‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
2+
AAD_TOKEN=$( az account get-access-token \
3+
--resource 2ff814a6-3304-4ab8-85cb-cd0e6f879c1d \
4+
--query "accessToken" \
5+
--output tsv )
6+
7+
echo $AAD_TOKEN
8+
9+
CREATE_REPO_RESPONSE=$(curl -X POST -H "Authorization: Bearer $AAD_TOKEN" \
10+
-H "X-Databricks-Azure-Workspace-Resource-Id: $WORKSPACE_ID" \
11+
-H 'Content-Type: application/scim+json' \
12+
-d $JSON_STRING \
13+
'{
14+
"displayName": "My Service Principal",
15+
"applicationId": "12a34b56-789c-0d12-e3fa-b456789c0123",
16+
"entitlements": [
17+
{
18+
"value": "allow-cluster-create"
19+
}
20+
],
21+
"schemas": [
22+
"urn:ietf:params:scim:schemas:core:2.0:ServicePrincipal"
23+
],
24+
"active": true
25+
}' https://$DATABRICKS_INSTANCE/api/2.0/preview/scim/v2/ServicePrincipals )
26+
27+
28+
29+
30+
31+

‎.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
az config set extension.use_dynamic_install=yes_without_prompt
2+
az extension add --name databricks
13

24
### Lets Retrieve Important Variables That Are Important For Later Steps
35

‎.azureDevOps/MLOps_Engineer/Utilities/Python/utilsGitConfiguration.py‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import json
1010

1111

12-
def configureGit(gitConfig, workspaceId, databricksInstance, bearerToken, managementToken, githubToken, environment):
12+
def configureGit(gitConfig, workspaceId, databricksInstance, bearerToken, managementToken, SYSTEM_ACCESSTOKEN ):
1313

1414
DBRKS_REQ_HEADERS = {
1515
'Authorization': f'Bearer {bearerToken}',
@@ -19,7 +19,7 @@ def configureGit(gitConfig, workspaceId, databricksInstance, bearerToken, manage
1919
}
2020

2121
newData = {
22-
"personal_access_token": githubToken
22+
"personal_access_token": SYSTEM_ACCESSTOKEN
2323
}
2424

2525
gitConfig.update(newData)
@@ -41,5 +41,4 @@ def configureGit(gitConfig, workspaceId, databricksInstance, bearerToken, manage
4141
databricksInstance=os.environ['DATABRICKS_INSTANCE'],
4242
bearerToken=os.environ['DBRKS_BEARER_TOKEN'],
4343
managementToken=os.environ['DBRKS_MANAGEMENT_TOKEN'],
44-
githubToken=os.environ['PAT_GITHUB'],
45-
environment=os.environ['ENVIRONMENT'])
44+
SYSTEM_ACCESSTOKEN=os.environ['SYSTEM_ACCESSTOKEN'] )

‎.azureDevOps/MLOps_Engineer/Variables/Development/RBAC.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@
44
"roles": [
55
"Key Vault Administrator"
66
],
7-
"roleBeneficiaryObjID": "3fb6e2d3-7734-43fc-be9e-af8671acf605",
7+
"roleBeneficiaryObjID": "888a0304-df2d-4e40-b0e8-48ea893539ef",
88
"Description": "Your Object ID",
99
"principalType": "User"
1010
},
1111
{
1212
"roles": [
1313
"Contributor",
14-
"DBX_Custom_Role_DSToolkit",
1514
"Key Vault Administrator",
1615
"Storage Blob Data Contributor"
1716
],
18-
"roleBeneficiaryObjID": "4f305dd8-567a-4db8-aaf6-cf8b3bdd9337",
17+
"roleBeneficiaryObjID": "587604dd-f10d-450b-8238-8d6a0a75371d",
1918
"Description": "Databricks SPN",
2019
"principalType": "ServicePrincipal"
2120
}

‎.azureDevOps/MLOps_Engineer/Variables/Production/RBAC.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@
44
"roles": [
55
"Key Vault Administrator"
66
],
7-
"roleBeneficiaryObjID": "3fb6e2d3-7734-43fc-be9e-af8671acf605",
7+
"roleBeneficiaryObjID": "888a0304-df2d-4e40-b0e8-48ea893539ef",
88
"Description": "Your Object ID",
99
"principalType": "User"
1010
},
1111
{
1212
"roles": [
1313
"Contributor",
14-
"DBX_Custom_Role_DSToolkit",
1514
"Key Vault Administrator",
1615
"Storage Blob Data Contributor"
1716
],
18-
"roleBeneficiaryObjID": "4f305dd8-567a-4db8-aaf6-cf8b3bdd9337",
17+
"roleBeneficiaryObjID": "587604dd-f10d-450b-8238-8d6a0a75371d",
1918
"Description": "Databricks SPN",
2019
"principalType": "ServicePrincipal"
2120
}

‎.azureDevOps/MLOps_Engineer/Variables/Sandbox/RBAC.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@
44
"roles": [
55
"Key Vault Administrator"
66
],
7-
"roleBeneficiaryObjID": "3fb6e2d3-7734-43fc-be9e-af8671acf605",
7+
"roleBeneficiaryObjID": "888a0304-df2d-4e40-b0e8-48ea893539ef",
88
"Description": "Your Object ID",
99
"principalType": "User"
1010
},
1111
{
1212
"roles": [
1313
"Contributor",
14-
"DBX_Custom_Role_DSToolkit",
1514
"Key Vault Administrator",
1615
"Storage Blob Data Contributor"
1716
],
18-
"roleBeneficiaryObjID": "4f305dd8-567a-4db8-aaf6-cf8b3bdd9337",
17+
"roleBeneficiaryObjID": "587604dd-f10d-450b-8238-8d6a0a75371d",
1918
"Description": "Databricks SPN",
2019
"principalType": "ServicePrincipal"
2120
}

‎.azureDevOps/MLOps_Engineer/Variables/UAT/RBAC.json‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,18 +4,17 @@
44
"roles": [
55
"Key Vault Administrator"
66
],
7-
"roleBeneficiaryObjID": "3fb6e2d3-7734-43fc-be9e-af8671acf605",
7+
"roleBeneficiaryObjID": "888a0304-df2d-4e40-b0e8-48ea893539ef",
88
"Description": "Your Object ID",
99
"principalType": "User"
1010
},
1111
{
1212
"roles": [
1313
"Contributor",
14-
"DBX_Custom_Role_DSToolkit",
1514
"Key Vault Administrator",
1615
"Storage Blob Data Contributor"
1716
],
18-
"roleBeneficiaryObjID": "4f305dd8-567a-4db8-aaf6-cf8b3bdd9337",
17+
"roleBeneficiaryObjID": "587604dd-f10d-450b-8238-8d6a0a75371d",
1918
"Description": "Databricks SPN",
2019
"principalType": "ServicePrincipal"
2120
}

‎.azureDevOps/Pipelines/Managed_Identity_Auth/1-Master/onDeploy.yaml‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
2+
name: Managed Identity MLOps Databricks Deployment
3+
4+
trigger: none
5+
pr: none
6+
7+
8+
9+
# TO DO : Retrofit This Code So You Can Choose The Environments You Want To Deploy. This Will be
10+
# Helpfull if a Release Pipeline Fails And We Need To Rewind The Commit And Redeploy
11+
12+
# Create A Condition That This Will Not Run When A Pull Request Is Launched. Presumably a CI only Condition
13+
# After The First Environment Deployment, This YAML Pipeline is Triggering Alongside onRealease. I have Set
14+
# pr == none For Now To Prevent This Behaviour
15+
16+
17+
pool:
18+
name: 'vmss-linux-pool'
19+
20+
21+
parameters:
22+
23+
- name: ENVIRONMENT
24+
displayName: Choose Environment
25+
default: GenesisDeployment
26+
type: string
27+
values:
28+
- Sandbox
29+
- Development
30+
- UAT
31+
- Production
32+
- GenesisDeployment
33+
34+
35+
- name: MI_VMSS_NAME
36+
displayName: Enter name of Virtual Machine Scale Set With Managed Identity On Sub Enabled
37+
default: vmsslinux
38+
type: string
39+
40+
41+
- name: azureSubscription
42+
displayName: Enter Service Connection Name
43+
default: mi-vmss-spn
44+
type: string
45+
46+
47+
- name: PULL_BRANCH
48+
displayName: Branch For DBX Repo Folders
49+
default: ciarand/dbx_pipeline
50+
type: string
51+
52+
53+
stages:
54+
55+
- stage: SandboxDeploy
56+
condition: or(eq('${{ parameters.ENVIRONMENT }}', 'GenesisDeployment'), eq('${{ parameters.ENVIRONMENT }}', 'Sandbox'))
57+
displayName: SandboxDeploy
58+
jobs:
59+
- template: ..\2-Jobs\jobDatabricks.yaml
60+
parameters:
61+
Environment: Sandbox
62+
azureSubscription: ${{ parameters.azureSubscription }}
63+
enableRepoPull: false
64+
branchName: main
65+
MI_VMSS_NAME: ${{ parameters.MI_VMSS_NAME }}
66+
67+
- stage: DevelopmentDeploy
68+
condition: or(eq('${{ parameters.ENVIRONMENT }}', 'GenesisDeployment'), eq('${{ parameters.ENVIRONMENT }}', 'Development'))
69+
displayName: DevelopmentDeploy
70+
dependsOn: []
71+
jobs:
72+
- template: ..\2-Jobs\jobDatabricks.yaml
73+
parameters:
74+
Environment: Development
75+
azureSubscription: ${{ parameters.azureSubscription }}
76+
enableRepoPull: false
77+
branchName: main
78+
updateFolder: None
79+
MI_VMSS_NAME: ${{ parameters.MI_VMSS_NAME }}
80+
81+
- stage: UATDeploy
82+
condition: or(eq('${{ parameters.ENVIRONMENT }}', 'GenesisDeployment'), eq('${{ parameters.ENVIRONMENT }}', 'UAT'))
83+
displayName: UATDeploy
84+
dependsOn: []
85+
jobs:
86+
- template: ..\2-Jobs\jobDatabricks.yaml
87+
parameters:
88+
Environment: UAT
89+
azureSubscription: ${{ parameters.azureSubscription }}
90+
enableRepoPull: false
91+
branchName: main
92+
releaseBranch: 'release/1'
93+
MI_VMSS_NAME: ${{ parameters.MI_VMSS_NAME }}
94+
95+

0 commit comments

Comments
 (0)