updated release notes

Author: KshitizLohia

ads/feature_store/common/enums.py

@@ -295,6 +295,7 @@ class FeatureType(Enum):
     STRING_BINARY_MAP = "STRING_BINARY_MAP"
     STRING_BOOLEAN_MAP = "STRING_BOOLEAN_MAP"
     UNKNOWN = "UNKNOWN"
+    COMPLEX = "COMPLEX"
 
 
 class EntityType(Enum):

ads/feature_store/common/spark_session_singleton.py

@@ -9,7 +9,6 @@
 
 from ads.common.decorator.runtime_dependency import OptionalDependency
 import os
-
 from ads.common.oci_client import OCIClientFactory
 from ads.feature_store.common.utils.utility import get_env_bool
 
@@ -33,6 +32,31 @@
     raise
 
 
+def get_env_bool(env_var: str, default: bool = False) -> bool:
+    """
+    :param env_var: Environment variable name
+    :param default: Default environment variable value
+    :return: Value of the boolean env variable
+    """
+    env_val = os.getenv(env_var)
+    if env_val is None:
+        env_val = default
+    else:
+        env_val = env_val.lower()
+        if env_val == "true":
+            env_val = True
+        elif env_val == "false":
+            env_val = False
+        else:
+            raise ValueError(
+                "For environment variable: {0} only string values T/true or F/false are allowed but: \
+                {1} was provided.".format(
+                    env_var, env_val
+                )
+            )
+    return env_val
+
+
 def developer_enabled():
     return get_env_bool("DEVELOPER_MODE", False)
 
@@ -76,6 +100,8 @@ def __init__(self, metastore_id: str = None):
                 "spark.hadoop.oracle.dcat.metastore.id", metastore_id
             ).config(
                 "spark.sql.warehouse.dir", metastore.default_managed_table_location
+            ).config(
+                "spark.driver.memory", "16G"
             )
 
         if developer_enabled():

ads/feature_store/common/utils/feature_schema_mapper.py

@@ -71,7 +71,7 @@ def map_spark_type_to_feature_type(spark_type):
     if spark_type in spark_type_to_feature_type:
         return spark_type_to_feature_type.get(spark_type)
     else:
-        return FeatureType.UNKNOWN
+        return FeatureType.COMPLEX
 
 
 def map_pandas_type_to_feature_type(feature_name, values):
@@ -180,7 +180,7 @@ def map_feature_type_to_spark_type(feature_type):
     if feature_type_in in spark_types:
         return spark_types.get(feature_type_in)
     else:
-        return "UNKNOWN"
+        return "COMPLEX"
 
 
 def get_raw_data_source_schema(raw_feature_details: List[dict]):
@@ -225,31 +225,22 @@ def map_feature_type_to_pandas(feature_type):
         FeatureType.INTEGER: "int32",
         FeatureType.DECIMAL: "object",
         FeatureType.DATE: "object",
+        FeatureType.STRING_ARRAY: "object",
+        FeatureType.INTEGER_ARRAY: "object",
+        FeatureType.LONG_ARRAY: "object",
+        FeatureType.FLOAT_ARRAY: "object",
+        FeatureType.DOUBLE_ARRAY: "object",
+        FeatureType.TIMESTAMP_ARRAY: "object",
+        FeatureType.BOOLEAN_ARRAY: "object",
+        # FeatureType.DECIMAL_ARRAY: "object",
+        FeatureType.DATE_ARRAY: "object",
     }
     if feature_type_in in supported_feature_type:
         return supported_feature_type.get(feature_type_in)
     else:
         raise TypeError(f"Feature Type {feature_type} is not supported for pandas")
 
 
-def convert_pandas_datatype_with_schema(
-    raw_feature_details: List[dict], input_df: pd.DataFrame
-):
-    feature_detail_map = {}
-    for feature_details in raw_feature_details:
-        feature_detail_map[feature_details.get("name")] = feature_details
-    for column in input_df.columns:
-        if column in feature_detail_map.keys():
-            feature_details = feature_detail_map[column]
-            feature_type = feature_details.get("featureType")
-            pandas_type = map_feature_type_to_pandas(feature_type)
-            input_df[column] = (
-                input_df[column]
-                .astype(pandas_type)
-                .where(pd.notnull(input_df[column]), None)
-            )
-
-
 def map_spark_type_to_stats_data_type(spark_type):
     """Maps the spark data types to MLM library data types
     args: