adding boxplot, fixing alignment issues and supporting per feature statistic visualisation

Author: harsh97

ads/feature_store/dataset.py

@@ -38,7 +38,7 @@
 from ads.feature_store.feature_group_expectation import Expectation
 from ads.feature_store.feature_option_details import FeatureOptionDetails
 from ads.feature_store.service.oci_dataset import OCIDataset
-from ads.feature_store.statistics import Statistics
+from ads.feature_store.statistics.statistics import Statistics
 from ads.feature_store.statistics_config import StatisticsConfig
 from ads.feature_store.service.oci_lineage import OCILineage
 from ads.feature_store.model_details import ModelDetails

ads/feature_store/feature_group.py

@@ -44,7 +44,7 @@
 from ads.feature_store.service.oci_feature_group import OCIFeatureGroup
 from ads.feature_store.service.oci_feature_group_job import OCIFeatureGroupJob
 from ads.feature_store.service.oci_lineage import OCILineage
-from ads.feature_store.statistics import Statistics
+from ads.feature_store.statistics.statistics import Statistics
 from ads.feature_store.statistics_config import StatisticsConfig
 from ads.feature_store.validation_output import ValidationOutput
 

ads/feature_store/statistics/charts/box_plot.py

@@ -2,9 +2,13 @@
 # -*- coding: utf-8; -*-
 # Copyright (c) 2023 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
+from typing import List
 
 from ads.common.decorator.runtime_dependency import OptionalDependency
 from ads.feature_store.statistics.charts.abstract_feature_stat import AbsFeatureStat
+from ads.feature_store.statistics.charts.frequency_distribution import (
+    FrequencyDistribution,
+)
 from ads.feature_store.statistics.generic_feature_value import GenericFeatureValue
 
 try:
@@ -23,6 +27,8 @@ class BoxPlot(AbsFeatureStat):
     CONST_SD = "StandardDeviation"
     CONST_MEAN = "Mean"
     CONST_BOX_PLOT_TITLE = "Box Plot"
+    CONST_IQR = "IQR"
+    CONST_FREQUENCY_DISTRIBUTION = "FrequencyDistribution"
 
     class Quartiles:
         CONST_Q1 = "q1"
@@ -52,16 +58,15 @@ def __init__(
         sd: float,
         q1: float,
         q3: float,
-        min: float,
-        max: float,
+        boxpoints: List[float],
     ):
         self.mean = mean
         self.median = median
         self.q1 = q1
         self.q3 = q3
         self.sd = sd
-        self.min = min
-        self.max = max
+        self.iqr = self.q3 - self.q1
+        self.boxpoints = boxpoints
 
     def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
         xaxis_str, yaxis_str, x_str, y_str = self.get_x_y_str_axes(xaxis, yaxis)
@@ -71,14 +76,30 @@ def add_to_figure(self, fig: Figure, xaxis: int, yaxis: int):
             q1=[self.q1],
             q3=[self.q3],
             sd=[self.sd],
-            upperfence=[self.max],
-            lowerfence=[self.min],
+            y=[self.boxpoints],
+            upperfence=[self.q3 + 1.5 * self.iqr],
+            lowerfence=[self.q1 - 1.5 * self.iqr],
             xaxis=x_str,
             yaxis=y_str,
+            name="",
+            jitter=0,
         )
         fig.layout.annotations[xaxis].text = self.CONST_BOX_PLOT_TITLE
         fig.layout[yaxis_str]["title"] = "Values"
 
+    @staticmethod
+    def get_boxpoints_from_frequency_distribution(
+        frequency_distribution: FrequencyDistribution,
+    ) -> List[float]:
+        boxpoints = []
+        if frequency_distribution is not None:
+            for frequency, bin in zip(
+                frequency_distribution.frequency, frequency_distribution.bins
+            ):
+                boxpoints.extend([bin] * frequency)
+
+        return boxpoints
+
     @classmethod
     def from_json(cls, json_dict: dict) -> "BoxPlot":
         if type(json_dict) is dict and json_dict.get(cls.CONST_QUARTILES) is not None:
@@ -89,8 +110,11 @@ def from_json(cls, json_dict: dict) -> "BoxPlot":
                 sd=GenericFeatureValue.from_json(json_dict.get(cls.CONST_SD)).val,
                 q1=quartiles.q1,
                 q3=quartiles.q3,
-                min=GenericFeatureValue.from_json(json_dict.get(cls.CONST_MIN)).val,
-                max=GenericFeatureValue.from_json(json_dict.get(cls.CONST_MAX)).val,
+                boxpoints=cls.get_boxpoints_from_frequency_distribution(
+                    FrequencyDistribution.from_json(
+                        json_dict.get(cls.CONST_FREQUENCY_DISTRIBUTION)
+                    )
+                ),
             )
         else:
             return None

ads/feature_store/statistics/feature_stat.py

@@ -39,10 +39,10 @@ class FeatureStatistics:
     def __init__(
         self,
         feature_name: str,
-        top_k_frequent_elements: TopKFrequentElements,
-        frequency_distribution: FrequencyDistribution,
-        probability_distribution: ProbabilityDistribution,
-        box_plot: BoxPlot,
+        top_k_frequent_elements: TopKFrequentElements = None,
+        frequency_distribution: FrequencyDistribution = None,
+        probability_distribution: ProbabilityDistribution = None,
+        box_plot: BoxPlot = None,
     ):
         self.feature_name: str = feature_name
         self.top_k_frequent_elements = top_k_frequent_elements
@@ -65,7 +65,7 @@ def from_json(cls, feature_name: str, json_dict: dict) -> "FeatureStatistics":
                 BoxPlot.from_json(json_dict),
             )
         else:
-            return None
+            return cls(feature_name)
 
     @property
     def __feature_stat_objects__(self) -> List[AbsFeatureStat]:
@@ -81,11 +81,18 @@ def __feature_stat_objects__(self) -> List[AbsFeatureStat]:
         ]
 
     def to_viz(self):
+        # TODO: make it generic
+        def next_graph_position_generator():
+            yield 1
+            yield 0
+            yield 2
+
         graph_count = len(self.__feature_stat_objects__)
         if graph_count > 0:
-            fig = make_subplots(cols=graph_count, column_titles=["title"] * graph_count)
-            for idx, stat in enumerate(
-                [stat for stat in self.__feature_stat_objects__ if stat is not None]
+            fig = make_subplots(cols=3, column_titles=[" "] * 3)
+            for idx, stat in zip(
+                next_graph_position_generator(),
+                [stat for stat in self.__feature_stat_objects__ if stat is not None],
             ):
                 stat.add_to_figure(fig, idx, idx)
 

ads/feature_store/statistics/statistics.py

@@ -32,6 +32,5 @@ def to_viz(self, feature_list: List[str] = None):
             [
                 FeatureStatistics.from_json(feature, stat).to_viz()
                 for feature, stat in stats.items()
-                if FeatureStatistics.from_json(feature, stat) is not None
-                and (feature_list is None or feature in feature_list)
+                if (feature_list is None or feature in feature_list)
             ]