Optimised report loading for anomaly operator

Author: govarsha

ads/opctl/operator/lowcode/anomaly/model/base_model.py

@@ -79,7 +79,7 @@ def generate_report(self):
                 anomaly_output, test_data, elapsed_time
             )
         table_blocks = [
-            rc.DataTable(df, label=col, index=True)
+            rc.DataTable(df.head(1000) if self.spec.optimize_report and len(df) > 1000 else df, label=col, index=True)
             for col, df in self.datasets.full_data_dict.items()
         ]
         data_table = rc.Select(blocks=table_blocks)
@@ -94,20 +94,35 @@ def generate_report(self):
             anomaly_col = anomaly_output.get_anomalies_by_cat(category=target)[
                 OutputColumns.ANOMALY_COL
             ]
+            anomaly_indices = [i for i, index in enumerate(anomaly_col) if index == 1]
+            downsampled_time_col = time_col
+            selected_indices = list(range(len(time_col)))
+            if self.spec.optimize_report:
+                non_anomaly_indices = [i for i in range(len(time_col)) if i not in anomaly_indices]
+                # Downsample non-anomalous data if it exceeds the threshold (1000)
+                if len(non_anomaly_indices) > 1000:
+                    downsampled_non_anomaly_indices = non_anomaly_indices[::len(non_anomaly_indices)//1000]
+                    selected_indices = sorted(anomaly_indices + downsampled_non_anomaly_indices)
+                downsampled_time_col = time_col[selected_indices]
+
             columns = set(df.columns).difference({date_column})
             for col in columns:
                 y = df[col].reset_index(drop=True)
+
+                downsampled_y = y[selected_indices]
+
                 fig, ax = plt.subplots(figsize=(8, 3), layout="constrained")
                 ax.grid()
-                ax.plot(time_col, y, color="black")
-                for i, index in enumerate(anomaly_col):
-                    if index == 1:
-                        ax.scatter(time_col[i], y[i], color="red", marker="o")
+                ax.plot(downsampled_time_col, downsampled_y, color="black")
+                # Plot anomalies
+                for i in anomaly_indices:
+                    ax.scatter(time_col[i], y[i], color="red", marker="o")
                 plt.xlabel(date_column)
                 plt.ylabel(col)
                 plt.title(f"`{col}` with reference to anomalies")
                 figure_blocks.append(rc.Widget(ax))
-            blocks.append(rc.Group(*figure_blocks, label=target))
+
+        blocks.append(rc.Group(*figure_blocks, label=target))
         plots = rc.Select(blocks)
 
         report_sections = []

ads/opctl/operator/lowcode/anomaly/operator_config.py

@@ -77,6 +77,7 @@ class AnomalyOperatorSpec(DataClassSerializable):
     model: str = None
     model_kwargs: Dict = field(default_factory=dict)
     contamination: float = None
+    optimize_report: bool = None
 
     def __post_init__(self):
         """Adjusts the specification details."""

ads/opctl/operator/lowcode/anomaly/schema.yaml

@@ -377,4 +377,9 @@ spec:
       type: dict
       required: false
 
+    optimize_report:
+      type: boolean
+      required: false
+      default: true
+
   type: dict

anomaly_out/anomali_2_test.csv

@@ -0,0 +1,8 @@
+PPG_Code,last_day_of_week,anomaly
+Product Group 107,2019-01-05,0
+Product Group 107,2019-01-12,0
+Product Group 107,2019-01-19,0
+Product Group 107,2019-01-26,0
+Product Group 107,2019-04-27,1
+Product Group 108,2019-05-04,1
+Product Group 108,2019-05-11,0