report added

Author: prasankh

ads/opctl/operator/lowcode/recommender/constant.py

@@ -18,6 +18,7 @@ class OutputColumns(str, metaclass=ExtendedEnumMeta):
 class SupportedMetrics(str, metaclass=ExtendedEnumMeta):
     """Supported forecast metrics."""
     RMSE = "RMSE"
+    MAE = "MAE"
 
 class SupportedModels(str, metaclass=ExtendedEnumMeta):
     """Supported recommender models."""

ads/opctl/operator/lowcode/recommender/model/base_model.py

@@ -5,39 +5,144 @@
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import os
+import tempfile
 import time
 from abc import ABC, abstractmethod
+from typing import Tuple, Dict
 
+import fsspec
 import pandas as pd
+import report_creator as rc
 
 from ads.common.object_storage_details import ObjectStorageDetails
 from ads.opctl import logger
 from ads.opctl.operator.lowcode.common.utils import default_signer
 from ads.opctl.operator.lowcode.common.utils import (
+    human_time_friendly,
+    enable_print,
+    disable_print,
     write_data,
 )
+from .factory import SupportedModels
 from .recommender_dataset import RecommenderDatasets
 from ..operator_config import RecommenderOperatorConfig
+from plotly import graph_objects as go
+import matplotlib.pyplot as plt
 
 
 class RecommenderOperatorBaseModel(ABC):
     """The base class for the recommender detection operator models."""
 
     def __init__(self, config: RecommenderOperatorConfig, datasets: RecommenderDatasets):
-        self.spec = config.spec
+        self.config = config
+        self.spec = self.config.spec
         self.datasets = datasets
 
     def generate_report(self):
+        item_col = self.spec.item_column
+        user_col = self.spec.user_column
+        interaction_col = self.spec.interaction_column
         start_time = time.time()
-        result_df = self._build_model()
+        result_df, metrics = self._build_model()
         elapsed_time = time.time() - start_time
         logger.info("Building the models completed in %s seconds", elapsed_time)
+
+        if self.spec.generate_report:
+            # build the report
+            (
+                model_description,
+                other_sections,
+            ) = self._generate_report()
+
+            header_section = rc.Block(
+                rc.Heading("Recommender Report", level=1),
+                rc.Text(
+                    f"The recommendations was generated using {SupportedModels.SVD.upper()}. {model_description}"
+                ),
+                rc.Group(
+                    rc.Metric(
+                        heading="Recommendations was generated in ",
+                        value=human_time_friendly(elapsed_time),
+                    ),
+                    rc.Metric(
+                        heading="Num users",
+                        value=len(self.datasets.users),
+                    ),
+                    rc.Metric(
+                        heading="Num items",
+                        value=len(self.datasets.items),
+                    )
+                ),
+            )
+
+        summary = rc.Block(
+            header_section,
+        )
+        # user and item distributions in interactions
+        user_title = rc.Heading("User Statistics", level=2)
+        user_rating_counts = self.datasets.interactions[user_col].value_counts()
+        fig_user = go.Figure(data=[go.Histogram(x=user_rating_counts, nbinsx=100)])
+        fig_user.update_layout(
+            title=f'Distribution of the number of interactions by {user_col}',
+            xaxis_title=f'Number of {interaction_col}',
+            yaxis_title=f'Number of {user_col}',
+            bargap=0.2
+        )
+        item_title = rc.Heading("Item Statistics", level=2)
+        item_rating_counts = self.datasets.interactions[item_col].value_counts()
+        fig_item = go.Figure(data=[go.Histogram(x=item_rating_counts, nbinsx=100)])
+        fig_item.update_layout(
+            title=f'Distribution of the number of interactions by {item_col}',
+            xaxis_title=f'Number of {interaction_col}',
+            yaxis_title=f'Number of {item_col}',
+            bargap=0.2
+        )
+        result_heatmap_title = rc.Heading("Sample Recommendations", level=2)
+        sample_items = result_df[item_col].head(100).index
+        filtered_df = result_df[result_df[item_col].isin(sample_items)]
+        data = filtered_df.pivot(index=user_col, columns=item_col, values=interaction_col)
+        fig = go.Figure(data=go.Heatmap(
+            z=data.values,
+            x=data.columns,
+            y=data.index,
+            colorscale='Viridis'
+        ))
+        fig.update_layout(
+            title='Recommendation heatmap of User-Item Interactions (sample)',
+            width=1500,
+            height=800,
+            xaxis_title=item_col,
+            yaxis_title=user_col,
+            coloraxis_colorbar=dict(title=interaction_col)
+        )
+        plots = [user_title, rc.Widget(fig_user),
+                 item_title, rc.Widget(fig_item),
+                 result_heatmap_title, rc.Widget(fig)]
+
+        test_metrics_sections = [rc.DataTable(pd.DataFrame(metrics, index=[0]))]
+        yaml_appendix_title = rc.Heading("Reference: YAML File", level=2)
+        yaml_appendix = rc.Yaml(self.config.to_dict())
+        report_sections = (
+                [summary]
+                + plots
+                + test_metrics_sections
+                + other_sections
+                + [yaml_appendix_title, yaml_appendix]
+        )
+
         # save the report and result CSV
         self._save_report(
+            report_sections=report_sections,
             result_df=result_df
         )
 
-    def _save_report(self, result_df):
+    def _evaluation_metrics(self):
+        pass
+
+    def _test_data_evaluate_metrics(self):
+        pass
+
+    def _save_report(self, report_sections: Tuple, result_df: pd.DataFrame):
         """Saves resulting reports to the given folder."""
 
         unique_output_dir = self.spec.output_directory.url
@@ -47,7 +152,25 @@ def _save_report(self, result_df):
         else:
             storage_options = dict()
 
-        # forecast csv report
+            # report-creator html report
+            if self.spec.generate_report:
+                with tempfile.TemporaryDirectory() as temp_dir:
+                    report_local_path = os.path.join(temp_dir, "___report.html")
+                    disable_print()
+                    with rc.ReportCreator("My Report") as report:
+                        report.save(rc.Block(*report_sections), report_local_path)
+                    enable_print()
+
+                    report_path = os.path.join(unique_output_dir, self.spec.report_filename)
+                    with open(report_local_path) as f1:
+                        with fsspec.open(
+                                report_path,
+                                "w",
+                                **storage_options,
+                        ) as f2:
+                            f2.write(f1.read())
+
+        # recommender csv report
         write_data(
             data=result_df,
             filename=os.path.join(unique_output_dir, self.spec.recommendations_filename),
@@ -68,7 +191,7 @@ def _generate_report(self):
         """
 
     @abstractmethod
-    def _build_model(self) -> pd.DataFrame:
+    def _build_model(self) -> [pd.DataFrame, Dict]:
         """
         Build the model.
         The method that needs to be implemented on the particular model level.

ads/opctl/operator/lowcode/recommender/model/svd.py

@@ -1,17 +1,22 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*--
+from typing import Tuple, Dict, Any
 
 # Copyright (c) 2023, 2024 Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl/
 
 import pandas as pd
+from pandas import DataFrame
+
 from .recommender_dataset import RecommenderDatasets
 from ..operator_config import RecommenderOperatorConfig
 from .factory import RecommenderOperatorBaseModel
 from surprise import Dataset, Reader
 from surprise.model_selection import train_test_split
 from surprise import SVD
-from surprise import accuracy
+from surprise.accuracy import rmse, mae
+import report_creator as rc
+from ..constant import SupportedMetrics
 
 
 class SVDOperatorModel(RecommenderOperatorBaseModel):
@@ -22,38 +27,62 @@ def __init__(self, config: RecommenderOperatorConfig, datasets: RecommenderDatas
         self.interactions = datasets.interactions
         self.users = datasets.users
         self.items = datasets.items
-        self.user_id = config.spec.user_column_name
-        self.item_id = config.spec.item_column_name
-        self.rating_col = config.spec.ratings_column_name
+        self.user_id = config.spec.user_column
+        self.item_id = config.spec.item_column
+        self.interaction_column = config.spec.interaction_column
         self.test_size = 0.2
+        self.algo = SVD()
 
-    def _get_recommendations(self, user_id, algo, items, n=10):
-        all_item_ids = items[self.item_id].unique()
+    def _get_recommendations(self, user_id, n):
+        all_item_ids = self.items[self.item_id].unique()
         rated_items = self.interactions[self.interactions[self.user_id] == user_id][self.item_id]
         unrated_items = [item_id for item_id in all_item_ids if item_id not in rated_items.values]
-        predictions = [algo.predict(user_id, item_id) for item_id in unrated_items]
+        predictions = [self.algo.predict(user_id, item_id) for item_id in unrated_items]
         predictions.sort(key=lambda x: x.est, reverse=True)
         top_n_recommendations = predictions[:n]
         return [(pred.iid, pred.est) for pred in top_n_recommendations]
 
-    def _build_model(self) -> pd.DataFrame:
-        min_rating = self.interactions[self.rating_col].min()
-        max_rating = self.interactions[self.rating_col].max()
+    def _build_model(self) -> Tuple[DataFrame, Dict]:
+        min_rating = self.interactions[self.interaction_column].min()
+        max_rating = self.interactions[self.interaction_column].max()
         reader = Reader(rating_scale=(min_rating, max_rating))
-        data = Dataset.load_from_df(self.interactions[[self.user_id, self.item_id, self.rating_col]], reader)
+        data = Dataset.load_from_df(self.interactions[[self.user_id, self.item_id, self.interaction_column]], reader)
         trainset, testset = train_test_split(data, test_size=self.test_size)
-        algo = SVD()
-        algo.fit(trainset)
-        predictions = algo.test(testset)
-        accuracy.rmse(predictions)
+        self.algo.fit(trainset)
+        predictions = self.algo.test(testset)
+
+        metric = {}
+        metric[SupportedMetrics.RMSE] = rmse(predictions, verbose=True)
+        metric[SupportedMetrics.MAE] = mae(predictions, verbose=True)
         all_recommendations = []
         for user_id in self.users[self.user_id]:
-            recommendations = self._get_recommendations(user_id, algo, self.items, n=self.spec.top_k)
+            recommendations = self._get_recommendations(user_id, n=self.spec.top_k)
             for item_id, est_rating in recommendations:
                 all_recommendations.append({
                     self.user_id: user_id,
                     self.item_id: item_id,
-                    self.rating_col: est_rating
+                    self.interaction_column: est_rating
                 })
         recommendations_df = pd.DataFrame(all_recommendations)
-        return recommendations_df
+        return recommendations_df, metric
+
+    def _generate_report(self):
+        model_description = """
+            Singular Value Decomposition (SVD) is a matrix factorization technique used in recommendation systems to
+            decompose a user-item interaction matrix into three constituent matrices. These matrices capture the
+            latent factors that explain the observed interactions.
+            """
+        new_user_recommendations = self._get_recommendations("__new_user__", self.spec.top_k)
+        new_recommendations = []
+        for item_id, est_rating in new_user_recommendations:
+            new_recommendations.append({
+                self.user_id: "__new_user__",
+                self.item_id: item_id,
+                self.interaction_column: est_rating
+            })
+        title = rc.Heading("Recommendations for new users", level=2)
+        other_sections = [title, rc.DataTable(new_recommendations)]
+        return (
+            model_description,
+            other_sections
+        )

ads/opctl/operator/lowcode/recommender/operator_config.py

@@ -32,16 +32,24 @@ class RecommenderOperatorSpec(DataClassSerializable):
     output_directory: OutputDirectory = field(default_factory=OutputDirectory)
     top_k: int = None
     model_name: str = None
-    user_column_name: str = None
-    item_column_name: str = None
-    ratings_column_name: str = None
+    user_column: str = None
+    item_column: str = None
+    interaction_column: str = None
     recommendations_filename: str = None
+    generate_report: bool = None
+    report_filename: str = None
+
 
     def __post_init__(self):
         """Adjusts the specification details."""
         self.output_directory = self.output_directory or OutputDirectory(url=find_output_dirname(self.output_directory))
         self.model_name = self.model_name or SupportedModels.SVD
         self.recommendations_filename = self.recommendations_filename or "recommendations.csv"
+        # For Report Generation. When user doesn't specify defaults to True
+        self.generate_report = (
+            self.generate_report if self.generate_report is not None else True
+        )
+        self.report_filename = self.report_filename or "report.html"
 
 
 @dataclass(repr=True)

ads/opctl/operator/lowcode/recommender/schema.yaml

@@ -246,17 +246,17 @@ spec:
       meta:
         description: "Placed into output_directory location. Defaults to report.html"
 
-    user_column_name:
+    user_column:
       type: string
       required: true
       default: "user_id"
 
-    item_column_name:
+    item_column:
       type: string
       required: true
       default: "item_id"
 
-    rating_column_name:
+    interaction_column:
       type: string
       required: true
       default: "rating"