Skip to content

Commit 8222351

Browse files
Xinyao Liuganbaras
Xinyao Liu
authored and
ganbaras
committed
Added - Support for OCI Data Science - ML Pipelines - Parameterized pipelines
1 parent ac84863 commit 8222351

21 files changed

+1599
-342
lines changed

‎examples/datascience/job/job.tf‎

Lines changed: 68 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -46,16 +46,16 @@ resource "oci_datascience_project" "job" {
4646
compartment_id = var.compartment_ocid
4747
}
4848

49-
resource "oci_core_subnet" "tf_subnet" {
50-
cidr_block = "10.0.1.0/24"
51-
compartment_id = var.compartment_ocid
52-
vcn_id = oci_core_vcn.tf_vcn.id
53-
}
49+
#resource "oci_core_subnet" "tf_subnet" {
50+
# cidr_block = "10.0.1.0/24"
51+
# compartment_id = var.compartment_ocid
52+
# vcn_id = oci_core_vcn.tf_vcn.id
53+
#}
5454

55-
resource "oci_core_vcn" "tf_vcn" {
56-
cidr_block = "10.0.0.0/16"
57-
compartment_id = var.compartment_ocid
58-
}
55+
#resource "oci_core_vcn" "tf_vcn" {
56+
# cidr_block = "10.0.0.0/16"
57+
# compartment_id = var.compartment_ocid
58+
#}
5959

6060
resource "oci_datascience_job" "job" {
6161
compartment_id = var.compartment_ocid
@@ -75,69 +75,70 @@ resource "oci_datascience_job" "job" {
7575
# optional parameter
7676
# opc_parent_rpt_url = ""
7777

78-
# job_configuration_details {
79-
# job_type = "DEFAULT"
80-
# maximum_runtime_in_minutes = 30
81-
# }
78+
job_configuration_details {
79+
job_type = "DEFAULT"
80+
maximum_runtime_in_minutes = 30
81+
}
8282

83-
# job_infrastructure_configuration_details {
84-
# job_infrastructure_type = "STANDALONE"
85-
# shape_name = "VM.Standard3.Flex"
86-
# block_storage_size_in_gbs = 100
87-
# subnet_id = oci_core_subnet.tf_subnet.id
83+
job_infrastructure_configuration_details {
84+
job_infrastructure_type = "STANDALONE"
85+
shape_name = "VM.Standard3.Flex"
86+
block_storage_size_in_gbs = 100
87+
subnet_id = "<subnet_id>"
8888

8989
# # Optional
90-
# job_shape_config_details {
91-
# memory_in_gbs = 16
92-
# ocpus = 2
93-
# }
94-
# }
90+
job_shape_config_details {
91+
memory_in_gbs = 16
92+
ocpus = 2
93+
}
94+
}
9595

9696
# New Optional parameter for Multi Node
97-
job_node_configuration_details {
98-
job_node_type = "MULTI_NODE"
99-
job_network_configuration {
100-
job_network_type = "CUSTOM_NETWORK"
101-
subnet_id= "<subnet_id>"
102-
}
103-
job_node_group_configuration_details_list {
104-
name= "replica1"
105-
job_configuration_details {
106-
job_type= "DEFAULT"
107-
command_line_arguments= "commandLineArguments"
108-
}
109-
job_infrastructure_configuration_details {
110-
job_infrastructure_type = "MULTI_NODE"
111-
shape_name = "VM.Standard3.Flex"
112-
block_storage_size_in_gbs = 50
113-
subnet_id = "<subnet_id>"
114-
115-
# Optional
116-
job_shape_config_details {
117-
memory_in_gbs = 16
118-
ocpus = 3
119-
}
120-
}
121-
minimum_success_replicas=1
122-
replicas=1
123-
}
124-
startup_order = "IN_ORDER"
125-
}
126-
127-
job_environment_configuration_details {
128-
cmd = var.job_environment_configuration_details_cmd
129-
entrypoint = var.job_environment_configuration_details_entrypoint
130-
image = var.job_environment_configuration_details_image
131-
image_digest = var.job_environment_configuration_details_image_digest
132-
image_signature_id = var.job_environment_configuration_details_image_signature_id
133-
job_environment_type = var.job_environment_configuration_details_job_environment_type
134-
}
135-
136-
job_log_configuration_details {
137-
enable_logging = true
138-
enable_auto_log_creation = true
139-
log_group_id = oci_logging_log_group.job.id
140-
}
97+
# job_node_configuration_details {
98+
# job_node_type = "MULTI_NODE"
99+
# job_network_configuration {
100+
# job_network_type = "CUSTOM_NETWORK"
101+
# subnet_id= "<subnet_id>"
102+
# }
103+
# job_node_group_configuration_details_list {
104+
# name= "replica1"
105+
# job_configuration_details {
106+
# job_type= "DEFAULT"
107+
# command_line_arguments= "commandLineArguments"
108+
# }
109+
# job_infrastructure_configuration_details {
110+
# job_infrastructure_type = "MULTI_NODE"
111+
# shape_name = "VM.Standard3.Flex"
112+
# block_storage_size_in_gbs = 50
113+
# # subnet_id = "<subnet_id>"
114+
#
115+
# # Optional
116+
# job_shape_config_details {
117+
# memory_in_gbs = 16
118+
# ocpus = 3
119+
# cpu_baseline = "BASELINE_1_2"
120+
# }
121+
# }
122+
# minimum_success_replicas=1
123+
# replicas=1
124+
# }
125+
# startup_order = "IN_ORDER"
126+
# }
127+
128+
# job_environment_configuration_details {
129+
# cmd = var.job_environment_configuration_details_cmd
130+
# entrypoint = var.job_environment_configuration_details_entrypoint
131+
# image = var.job_environment_configuration_details_image
132+
# image_digest = var.job_environment_configuration_details_image_digest
133+
# image_signature_id = var.job_environment_configuration_details_image_signature_id
134+
# job_environment_type = var.job_environment_configuration_details_job_environment_type
135+
# }
136+
137+
# job_log_configuration_details {
138+
# enable_logging = true
139+
# enable_auto_log_creation = true
140+
# log_group_id = oci_logging_log_group.job.id
141+
# }
141142
}
142143

143144
data "oci_datascience_jobs" "by_compartment" {

‎examples/datascience/parameterized_pipeline_test.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/usr/bin/env python
2+
import json
3+
4+
try:
5+
# Define the data to write
6+
data = {
7+
"ocpus": "2",
8+
"memory": "32",
9+
"flexShape": "VM.Standard.E5.Flex",
10+
"nonFlexShape": "VM.Standard2.1",
11+
"blockStorage": "100"
12+
}
13+
14+
# Write to the JSON file
15+
with open("/home/datascience/output.json", 'w') as f:
16+
json.dump(data, f, indent=2)
17+
print(f"Successfully wrote data to /home/datascience/output.json")
18+
19+
except Exception as e:
20+
print(f"An error occurred: {str(e)}")

0 commit comments

Comments
 (0)