dataset changes

Author: najiyacl

ads/feature_store/dataset.py

@@ -1101,7 +1101,12 @@ def get_statistics(self, job_id: str = None) -> "Statistics":
         feature_statistics = (
             output_details.get("featureStatistics") if output_details else None
         )
-        return Statistics(feature_statistics)
+        stat_version = (
+            output_details.get("version") if output_details else None
+        )
+        version = stat_version if stat_version is not None else 1
+
+        return Statistics(feature_statistics, version)
 
     def get_validation_output(self, job_id: str = None) -> "ValidationOutput":
         """Retrieve Statistics object for the job with job_id

ads/feature_store/execution_strategy/spark/spark_execution.py

@@ -385,6 +385,7 @@ def _save_dataset_input(self, dataset, dataset_job: DatasetJob):
         validation_output = None
         feature_statistics = None
         output_features = []
+        version = 2  # after MLM upgrade
         database = (
             dataset.entity_id
         )  # Get the database and table name using entity_id and name of the dataset.
@@ -451,6 +452,7 @@ def _save_dataset_input(self, dataset, dataset_job: DatasetJob):
             "validation_output": str(validation_output) if validation_output else None,
             "commit_id": "commit_id",
             "feature_statistics": feature_statistics,
+            "version": version
         }
 
         self._update_job_and_parent_details(