ODSC-50530: Pack recommended infra config into operator yaml (#467)

Author: mrDzurb

ads/opctl/operator/cmd.py

@@ -208,15 +208,16 @@ def init(
 
         with fsspec.open(
             os.path.join(
-                output, f"{operator_info.type}_{'_'.join(key).replace('.','_')}.yaml"
+                output,
+                f"{operator_info.type}_{'_'.join(key).replace('.','_')}_backend.yaml",
             ),
             mode="w",
         ) as f:
             f.write(yaml.dump(tmp_config))
 
-    logger.info("#" * 100)
+    logger.info("#" * 50)
     logger.info(f"The auto-generated configs have been placed in: {output}")
-    logger.info("#" * 100)
+    logger.info("#" * 50)
 
 
 @runtime_dependency(module="docker", install_from=OptionalDependency.OPCTL)

ads/opctl/operator/common/backend_factory.py

@@ -419,7 +419,16 @@ def _init_backend_config(
 
                     # get config info from ini files
                     p = ConfigProcessor(
-                        {**runtime_kwargs, **{"execution": {"backend": resource_type}}}
+                        {
+                            **runtime_kwargs,
+                            **{"execution": {"backend": resource_type}},
+                            **{
+                                "infrastructure": {
+                                    **operator_info.jobs_default_params.to_dict(),
+                                    **operator_info.dataflow_default_params.to_dict(),
+                                }
+                            },
+                        }
                     ).step(
                         ConfigMerger,
                         ads_config=ads_config or DEFAULT_ADS_CONFIG_FOLDER,

ads/opctl/operator/common/operator_loader.py

@@ -13,7 +13,7 @@
 import sys
 import tempfile
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any, Dict, List
 from urllib.parse import urlparse
 
@@ -33,6 +33,74 @@
 LOCAL_SCHEME = "local"
 MAIN_BRANCH = "main"
 
+DEFAULT_SHAPE = "VM.Standard.E4.Flex"
+DEFAULT_OCPUS = 32
+DEFAULT_MEMORY_IN_GBS = 512
+DEFAULT_BLOCK_STORAGE_SIZE_IN_GBS = 512
+DEFAULT_SPARK_VERSION = "3.2.1"
+DEFAULT_NUM_OF_EXECUTORS = 1
+
+
+@dataclass(repr=True)
+class JobsDefaultParams(DataClassSerializable):
+    """Class representing the default params for the DataScience Job.
+
+    Attributes
+    ----------
+    shape_name (str)
+        The name of the shape.
+    ocpus (int)
+        The OCPUs count.
+    memory_in_gbs (int)
+        The size of the memory in GBs.
+    block_storage_size_in_GBs (int)
+        Size of the block storage drive.
+    """
+
+    shape_name: str = DEFAULT_SHAPE
+    ocpus: int = DEFAULT_OCPUS
+    memory_in_gbs: int = DEFAULT_MEMORY_IN_GBS
+    block_storage_size_in_GBs: int = DEFAULT_BLOCK_STORAGE_SIZE_IN_GBS
+
+    @classmethod
+    def from_dict(cls, *args, **kwargs: Dict) -> "JobsDefaultParams":
+        return super().from_dict(*args, **{**kwargs, **{"side_effect": None}})
+
+
+@dataclass(repr=True)
+class DataFlowDefaultParams(DataClassSerializable):
+    """Class representing the default params for the Data Flow Application.
+
+    Attributes
+    ----------
+    driver_shape (str)
+        The name of the driver shape.
+    driver_shape_ocpus (int)
+        The OCPUs count for the driver shape.
+    driver_shape_memory_in_gbs (int)
+        The size of the memory in GBs for the driver shape.
+    executor_shape (str)
+        The name of the executor shape.
+    executor_shape_ocpus (int)
+        The OCPUs count for the executor shape.
+    executor_shape_memory_in_gbs (int)
+        The size of the memory in GBs for the executor shape.
+    num_executors (int)
+        The number of executors.
+    spark_version (str)
+        The version of the SPARK.
+    """
+
+    spark_version: str = DEFAULT_SPARK_VERSION
+    driver_shape: str = DEFAULT_SHAPE
+    driver_shape_ocpus: int = DEFAULT_OCPUS
+    driver_shape_memory_in_gbs: int = DEFAULT_MEMORY_IN_GBS
+
+    num_executors: int = DEFAULT_NUM_OF_EXECUTORS
+    executor_shape: str = DEFAULT_SHAPE
+    executor_shape_ocpus: int = DEFAULT_OCPUS
+    executor_shape_memory_in_gbs: int = DEFAULT_MEMORY_IN_GBS
+
 
 @dataclass(repr=True)
 class OperatorInfo(DataClassSerializable):
@@ -57,11 +125,17 @@ class OperatorInfo(DataClassSerializable):
     conda_type (str)
         The type of conda pack (e.g., PACK_TYPE.CUSTOM).
     path (str)
-        The location of the operator.
+        The physical location of the operator.
     keywords (List[str])
         Keywords associated with the operator.
     backends (List[str])
         List of supported backends.
+    jobs_default_params (JobsDefaultParams)
+        The default params for the Jobs service.
+        Will be used when operator run on the Jobs service.
+    dataflow_default_params (DataFlowDefaultParams)
+        The default params for the DataFlow service.
+        Will be used when operator run on the DataFlow service.
 
     Properties
     ----------
@@ -79,6 +153,10 @@ class OperatorInfo(DataClassSerializable):
     path: str = ""
     keywords: List[str] = None
     backends: List[str] = None
+    jobs_default_params: JobsDefaultParams = field(default_factory=JobsDefaultParams)
+    dataflow_default_params: DataFlowDefaultParams = field(
+        default_factory=DataFlowDefaultParams
+    )
 
     @property
     def conda_prefix(self) -> str:
@@ -107,6 +185,10 @@ def __post_init__(self):
         self.version = self.version or "v1"
         self.conda_type = self.conda_type or PACK_TYPE.CUSTOM
         self.conda = self.conda or f"{self.type}_{self.version}"
+        self.jobs_default_params = self.jobs_default_params or JobsDefaultParams()
+        self.dataflow_default_params = (
+            self.dataflow_default_params or DataFlowDefaultParams()
+        )
 
     @classmethod
     def from_yaml(

ads/opctl/operator/common/operator_schema.yaml

@@ -56,3 +56,75 @@ conda:
   type: string
   meta:
     description: "The operator's conda environment name. Will be auto-generated if not provided."
+jobs_default_params:
+  required: false
+  type: dict
+  meta:
+    description: "The default parameters for the Data Science Job."
+  schema:
+    shape_name:
+      required: false
+      type: string
+      meta:
+        description: "The name of the shape."
+    ocpus:
+      required: false
+      type: integer
+      meta:
+        description: "The OCPUs count."
+    memory_in_gbs:
+      required: false
+      type: integer
+      meta:
+        description: "The size of the memory in GBs."
+    block_storage_size_in_GBs:
+      required: false
+      type: integer
+      meta:
+        description: "Size of the block storage drive."
+dataflow_default_params:
+  required: false
+  type: dict
+  meta:
+    description: "The default params for the DataFlow service."
+  schema:
+    spark_version:
+      required: false
+      type: string
+      meta:
+        description: "The version of the SPARK."
+    driver_shape:
+      required: false
+      type: string
+      meta:
+        description: "The name of the driver shape."
+    driver_shape_ocpus:
+      required: false
+      type: integer
+      meta:
+        description: "The OCPUs count for the driver shape."
+    driver_shape_memory_in_gbs:
+      required: false
+      type: integer
+      meta:
+        description: "The size of the memory in GBs for the driver shape."
+    executor_shape:
+      required: false
+      type: string
+      meta:
+        description: "The name of the executor shape."
+    executor_shape_ocpus:
+      required: false
+      type: integer
+      meta:
+        description: "The OCPUs count for the executor shape."
+    executor_shape_memory_in_gbs:
+      required: false
+      type: integer
+      meta:
+        description: "The size of the memory in GBs for the executor shape."
+    num_executors:
+      required: false
+      type: integer
+      meta:
+        description: "The number of executors."

ads/opctl/operator/lowcode/forecast/MLoperator

@@ -4,6 +4,11 @@ name: Forecasting Operator
 conda_type: published
 conda: forecast_v1
 gpu: no
+jobs_default_params:
+  shape_name: VM.Standard.E4.Flex
+  ocpus: 32
+  memory_in_gbs: 512
+  block_storage_size_in_GBs: 512
 keywords:
   - Prophet
   - AutoML

ads/opctl/operator/lowcode/forecast/README.md

@@ -6,11 +6,7 @@ Below are the steps to configure and run the Forecasting Operator on different r
 
 ## 1. Prerequisites
 
-Follow the [CLI Configuration](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/opctl/configure.html) steps from the ADS documentation. This step is mandatory as it sets up default values for different options while running the Forecasting Operator on OCI Data Science jobs or OCI Data Flow applications. If you have previously done this and used a flexible shape, make sure to adjust `ml_job_config.ini` with shape config details and `docker_registry` information.
-
-- ocpus = 1
-- memory_in_gbs = 16
-- docker_registry = `<iad.ocir.io/namespace/>`
+Follow the [CLI Configuration](https://accelerated-data-science.readthedocs.io/en/latest/user_guide/cli/opctl/configure.html) steps from the ADS documentation. This step is mandatory as it sets up default values for different options while running the Forecasting Operator on OCI Data Science Jobs.
 
 ## 2. Generating configs
 
@@ -23,10 +19,10 @@ ads operator init -t forecast --overwrite --output ~/forecast/
 The most important files expected to be generated are:
 
 - `forecast.yaml`: Contains forecast-related configuration.
-- `backend_operator_local_python_config.yaml`: This includes a local backend configuration for running forecasting in a local environment. The environment should be set up manually before running the operator.
-- `backend_operator_local_container_config.yaml`: This includes a local backend configuration for running forecasting within a local container. The container should be built before running the operator. Please refer to the instructions below for details on how to accomplish this.
-- `backend_job_container_config.yaml`: Contains Data Science job-related config to run forecasting in a Data Science job within a container (BYOC) runtime. The container should be built and published before running the operator. Please refer to the instructions below for details on how to accomplish this.
-- `backend_job_python_config.yaml`: Contains Data Science job-related config to run forecasting in a Data Science job within a conda runtime. The conda should be built and published before running the operator.
+- `forecast_operator_local_python_backend.yaml`: This includes a local backend configuration for running forecasting in a local environment. The environment should be set up manually before running the operator.
+- `forecast_operator_local_container_backend.yaml`: This includes a local backend configuration for running forecasting within a local container. The container should be built before running the operator. Please refer to the instructions below for details on how to accomplish this.
+- `forecast_job_container_backend.yaml`: Contains Data Science job-related config to run forecasting in a Data Science job within a container (BYOC) runtime. The container should be built and published before running the operator. Please refer to the instructions below for details on how to accomplish this.
+- `forecast_job_python_backend.yaml`: Contains Data Science job-related config to run forecasting in a Data Science job within a conda runtime. The conda should be built and published before running the operator.
 
 All generated configurations should be ready to use without the need for any additional adjustments. However, they are provided as starter kit configurations that can be customized as needed.