Adding feature store beta service

Author: harsh97

.pre-commit-config.yaml

@@ -8,6 +8,7 @@ repos:
     -   id: check-json
     -   id: check-merge-conflict
     -   id: check-yaml
+        exclude: feature_store_*.yaml
         args: ['--allow-multiple-documents']
     -   id: detect-private-key
     -   id: end-of-file-fixer
@@ -24,4 +25,4 @@ repos:
     -   id: rst-backticks
     -   id: rst-inline-touching-normal
 
-exclude: ^(docs/)
+exclude: ^(docs/)

ads/common/decorator/runtime_dependency.py

@@ -64,6 +64,9 @@ class OptionalDependency:
     OPTUNA = "oracle-ads[optuna]"
     SPARK = "oracle-ads[spark]"
     HUGGINGFACE = "oracle-ads[huggingface]"
+    GREAT_EXPECTATIONS = "oracle-ads[great-expectations]"
+    PYDEEQU = "oracle-ads[pydeequ]"
+    GRAPHVIZ = "oracle-ads[graphviz]"
 
 
 def runtime_dependency(

ads/common/oci_client.py

@@ -1,20 +1,23 @@
 #!/usr/bin/env python
 # -*- coding: utf-8; -*-
 
-# Copyright (c) 2021, 2022 Oracle and/or its affiliates.
+# Copyright (c) 2021, 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import logging
 
 from oci.ai_language import AIServiceLanguageClient
+from oci.data_catalog import DataCatalogClient
 from oci.data_flow import DataFlowClient
 from oci.data_labeling_service import DataLabelingManagementClient
 from oci.data_labeling_service_dataplane import DataLabelingClient
 from oci.data_science import DataScienceClient
 from oci.identity import IdentityClient
 from oci.object_storage import ObjectStorageClient
+from oci.resource_search import ResourceSearchClient
 from oci.secrets import SecretsClient
 from oci.vault import VaultsClient
+from oci.feature_store import FeatureStoreClient
 
 logger = logging.getLogger(__name__)
 
@@ -62,6 +65,9 @@ def _client_impl(self, client):
             "ai_language": AIServiceLanguageClient,
             "data_labeling_dp": DataLabelingClient,
             "data_labeling_cp": DataLabelingManagementClient,
+            "feature_store": FeatureStoreClient,
+            "resource_search": ResourceSearchClient,
+            "data_catalog": DataCatalogClient
         }
 
         assert (
@@ -124,6 +130,18 @@ def ai_language(self):
     def data_labeling_cp(self):
         return self.create_client("data_labeling_cp")
 
+    @property
+    def feature_store(self):
+        return self.create_client("feature_store")
+
     @property
     def data_labeling_dp(self):
         return self.create_client("data_labeling_dp")
+
+    @property
+    def resource_search(self):
+        return self.create_client("resource_search")
+
+    @property
+    def data_catalog(self):
+        return self.create_client("data_catalog")

ads/dataset/progress.py

@@ -41,7 +41,7 @@ def __init__(self, max_progress=100, description="Running", verbose=False):
             self.start_time = time.time()
             self.description = description
 
-    def update(self, description=None):
+    def update(self, description=None, n=1):
         """
         Updates the progress bar
         """
@@ -57,7 +57,7 @@ def update(self, description=None):
                 description if description is not None else self.description
             )
             self.start_time = time.time()
-        self.progress_bar.update(1)
+        self.progress_bar.update(n)
         if description is not None:
             self.progress_bar.set_description(description, refresh=True)
 

ads/feature_store/__init__.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*--
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+import logging
+
+logger = logging.getLogger(__name__)

ads/feature_store/common/__init__.py

@@ -0,0 +1,287 @@
+#!/usr/bin/env python
+# -*- coding: utf-8; -*-
+
+# Copyright (c) 2023 Oracle and/or its affiliates.
+# Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+
+from enum import Enum
+
+
+class JobStatus(Enum):
+    """
+    An enumeration that represents the supported Job status.
+
+    Attributes:
+        SUCCEEDED (str): A string representation of the state of Succeeded job.
+        FAILED (str): A string representation of the state of Failed job.
+        CODE_EXECUTION (str): A string representation of the state of CodeExecution job.
+
+    Methods:
+        None
+    """
+
+    SUCCEEDED = "SUCCEEDED"
+    FAILED = "FAILED"
+    CODE_EXECUTION = "CODE_EXECUTION"
+
+
+class LevelType(Enum):
+    """
+    An enumeration defining the different types of logging levels.
+
+    Attributes:
+        ERROR (str): A string representing the highest logging level, indicating an error in the program.
+        WARNING (str): A string representing a lower logging level, indicating a potential issue or warning in the program.
+    """
+
+    ERROR = "ERROR"
+    WARNING = "WARNING"
+
+
+class DatasetIngestionMode(Enum):
+    """
+    An enumeration defining the possible modes for ingesting datasets.
+
+    Attributes:
+        SQL (str): A string representing the SQL mode, which is used to ingest datasets using SQL.
+    """
+
+    SQL = "SQL"
+
+
+class IngestionMode(Enum):
+    """
+    An enumeration that represents the supported Ingestion Mode in feature store.
+
+    Attributes:
+        OVERWRITE (str): Ingestion mode to overwrite the data in the system.
+        APPEND (str): Ingestion mode to append the data in the system.
+        UPSERT (str): Ingestion mode to insert and update the data in the system.
+
+    Methods:
+        None
+    """
+
+    OVERWRITE = "OVERWRITE"
+    APPEND = "APPEND"
+    DEFAULT = "DEFAULT"
+    UPSERT = "UPSERT"
+
+
+class JoinType(Enum):
+    """Enumeration of supported SQL join types.
+
+    Attributes:
+        INNER: Inner join.
+        LEFT: Left join.
+        RIGHT: Right join.
+        FULL: Full outer join.
+        CROSS: Cross join.
+        LEFT_SEMI_JOIN: Left semi join.
+    """
+
+    INNER = "INNER"
+    LEFT = "LEFT"
+    RIGHT = "RIGHT"
+    FULL = "FULL"
+    CROSS = "CROSS"
+    LEFT_SEMI_JOIN = "LEFT_SEMI_JOIN"
+
+
+class ExecutionEngine(Enum):
+    """
+    An enumeration that represents the supported execution engines.
+
+    Attributes:
+        SPARK (str): A string representation of the Apache Spark execution engine.
+        PANDAS (str): A string representation of the Pandas execution engine.
+
+    Methods:
+        None
+    """
+
+    SPARK = "SPARK"
+    PANDAS = "PANDAS"
+
+
+class DataFrameType(Enum):
+    """
+    An enumeration that represents the supported DataFrame types.
+
+    Attributes:
+        SPARK (str): A string representation for spark Data frame type.
+        PANDAS (str): A string representation for pandas Data frame type.
+
+    Methods:
+        None
+    """
+
+    SPARK = "SPARK"
+    PANDAS = "PANDAS"
+
+
+class ValidationEngineType(Enum):
+    """
+    An enumeration that represents the supported validation engines.
+
+    Attributes:
+        GREAT_EXPECTATIONS (str): A string representation of the great expectation execution engine.
+
+    Methods:
+        None
+    """
+
+    GREAT_EXPECTATIONS = "GREAT_EXPECTATIONS"
+
+
+class FeatureStoreJobType(Enum):
+    """
+    An enumeration that represents the Job type.
+
+    Attributes:
+        FEATURE_GROUP_INGESTION (str): A string representing that job is feature group ingestion.
+        DATASET_INGESTION (str): A string representing that job is dataset ingestion.
+        FEATURE_GROUP_DELETION (str): A string representing that job is feature group deletion.
+        DATASET_DELETION (str): A string representing that job is dataset deletion.
+
+    Methods:
+        None
+    """
+
+    FEATURE_GROUP_INGESTION = "FEATURE_GROUP_INGESTION"
+    DATASET_INGESTION = "DATASET_INGESTION"
+    FEATURE_GROUP_DELETION = "FEATURE_GROUP_DELETION"
+    DATASET_DELETION = "DATASET_DELETION"
+
+
+class LifecycleState(Enum):
+    """
+    An enumeration that represents the lifecycle state of feature store resources.
+
+    Attributes:
+        ACTIVE (str): A string representing Active resource.
+        FAILED (str): A string representing Failed resource.
+        NEEDS_ATTENTION (str): A string representing needs_attention resource.
+
+    Methods:
+        None
+    """
+
+    ACTIVE = "ACTIVE"
+    FAILED = "FAILED"
+    NEEDS_ATTENTION = "NEEDS_ATTENTION"
+
+
+class JobConfigurationType(Enum):
+    """
+    An enumeration defining the different types of job configuration modes for Spark.
+
+    Attributes:
+        SPARK_BATCH_AUTOMATIC (str): A string representing automatic job configuration mode for Spark Batch jobs.
+        SPARK_BATCH_MANUAL (str): A string representing manual job configuration mode for Spark Batch jobs.
+    """
+
+    SPARK_BATCH_AUTOMATIC = "SPARK_BATCH_AUTOMATIC"
+    SPARK_BATCH_MANUAL = "SPARK_BATCH_MANUAL"
+
+
+class ExpectationType(Enum):
+    """
+    An enumeration of the available expectation types for a feature store.
+
+    Attributes:
+        STRICT (str): A strict expectation type.
+        LENIENT (str): A lenient expectation type.
+        NO_EXPECTATION (str): A no expectation type.
+
+    Methods:
+        None
+    """
+
+    STRICT = "STRICT"
+    LENIENT = "LENIENT"
+    NO_EXPECTATION = "NO_EXPECTATION"
+
+
+class TransformationMode(Enum):
+    """
+    An enumeration defining the different modes for data transformation.
+
+    Attributes:
+        SQL (str): A string representing the SQL mode, which is used to transform data using SQL queries.
+        PANDAS (str): A string representing the Pandas mode, which is used to transform data using the Pandas library.
+    """
+
+    SQL = "sql"
+    PANDAS = "pandas"
+
+
+class FilterOperators(Enum):
+    """
+    An enumeration defining the different comparison operators for data filtering.
+
+    Attributes:
+        GE (str): A string representing the greater than or equal to operator.
+        GT (str): A string representing the greater than operator.
+        NE (str): A string representing the not equals operator.
+        EQ (str): A string representing the equals operator.
+        LE (str): A string representing the less than or equal to operator.
+        LT (str): A string representing the less than operator.
+        IN (str): A string representing the in operator.
+        LK (str): A string representing the like operator.
+    """
+
+    GE = "GREATER_THAN_OR_EQUAL"
+    GT = "GREATER_THAN"
+    NE = "NOT_EQUALS"
+    EQ = "EQUALS"
+    LE = "LESS_THAN_OR_EQUAL"
+    LT = "LESS_THAN"
+    IN = "IN"
+    LK = "LIKE"
+
+
+class FeatureType(Enum):
+    """
+    An enumeration of the available feature types for a feature store.
+
+    Attributes:
+        STRING (str): A string feature type.
+        INTEGER (str): An integer feature type.
+        FLOAT (str): A float feature type.
+        DOUBLE (str): A double feature type.
+        BOOLEAN (str): A boolean feature type.
+        DATE (str): A date feature type.
+        TIMESTAMP (str): A timestamp feature type.
+        DECIMAL (str): A decimal feature type.
+        BINARY (str): A binary feature type.
+        ARRAY (str): An array feature type.
+        MAP (str): A map feature type.
+        STRUCT (str): A struct feature type.
+    """
+
+    STRING = "STRING"
+    INTEGER = "INTEGER"
+    FLOAT = "FLOAT"
+    DOUBLE = "DOUBLE"
+    BOOLEAN = "BOOLEAN"
+    DATE = "DATE"
+    TIMESTAMP = "TIMESTAMP"
+    DECIMAL = "DECIMAL"
+    BINARY = "BINARY"
+    ARRAY = "ARRAY"
+    MAP = "MAP"
+    STRUCT = "STRUCT"
+
+
+class EntityType(Enum):
+    """
+    An enumeration of the supported entity types.
+
+    Attributes:
+        FEATURE_GROUP (str): A string representing the feature group.
+        DATASET (str): An string representing the dataset.
+    """
+
+    FEATURE_GROUP = "FEATURE_GROUP"
+    DATASET = "DATASET"