Add Review and review metrics.

Author: florijanstamenkovic

labelbox/__init__.py

@@ -2,5 +2,5 @@
 
 from labelbox.client import Client
 from labelbox.schema import (
-    Project, Dataset, DataRow, Label, User, Organization, Task, LabelingFrontend,
-    AssetMetadata, Webhook)
+    Project, Dataset, DataRow, Label, Review, User, Organization, Task,
+    LabelingFrontend, AssetMetadata, Webhook)

labelbox/orm/query.py

@@ -592,3 +592,14 @@ def edit_webhook(webhook, topics, url, status):
             results_query_part(type(webhook)))
 
     return query_str, {}
+
+
+def project_review_metrics(project, net_score):
+    project_id_param = "project_id"
+    net_score_literal = "None" if net_score is None else net_score.name
+    query_str = """query ProjectReviewMetricsPyApi($%s: ID!){
+        project(where: {id:$%s})
+        {reviewMetrics {labelAggregate(netScore: %s) {count}}}
+    }""" % (project_id_param, project_id_param, net_score_literal)
+
+    return query_str, {project_id_param: project.uid}

labelbox/schema.py

@@ -1,4 +1,5 @@
 from datetime import datetime, timezone
+from enum import Enum, auto
 import json
 import logging
 from multiprocessing.dummy import Pool as ThreadPool
@@ -33,6 +34,7 @@ class Project(DbObject, Updateable, Deletable):
     datasets = Relationship.ToMany("Dataset", True)
     created_by = Relationship.ToOne("User", False, "created_by")
     organization = Relationship.ToOne("Organization", False)
+    reviews = Relationship.ToMany("Review", True)
     labeling_frontend = Relationship.ToOne("LabelingFrontend")
     labeling_frontend_options = Relationship.ToMany(
         "LabelingFrontendOptions", False, "labeling_frontend_options")
@@ -104,6 +106,20 @@ def export_labels(self, timeout_seconds=60):
     # TODO Mutable (fetched) attributes
     # ...many, define which are required for v0.1
 
+    def review_metrics(self, net_score):
+        """ Returns this Project's review metrics.
+        Args:
+            net_score (None or Review.NetScore): Indicates desired metric.
+        Return:
+            int, aggregation count of reviews for given net_score.
+        """
+        if net_score not in (None,) + tuple(Review.NetScore):
+            raise InvalidQueryError("Review metrics net score must be either None "
+                                    "or one of Review.NetScore values")
+        query_str, params = query.project_review_metrics(self, net_score)
+        res = self.client.execute(query_str, params)
+        return res["data"]["project"]["reviewMetrics"]["labelAggregate"]["count"]
+
     def setup(self, labeling_frontend, labeling_frontend_options):
         """ Finalizes the Project setup.
         Args:
@@ -344,6 +360,11 @@ def create_metadata(self, meta_type, meta_value):
 
 
 class Label(DbObject, Updateable, BulkDeletable):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.reviews.supports_filtering = False
+
     label = Field.String("label")
     seconds_to_label = Field.Float("seconds_to_label")
     agreement = Field.Float("agreement")
@@ -352,11 +373,39 @@ class Label(DbObject, Updateable, BulkDeletable):
 
     project = Relationship.ToOne("Project")
     data_row = Relationship.ToOne("DataRow")
+    reviews = Relationship.ToMany("Review", False)
 
     @staticmethod
     def bulk_delete(objects):
         BulkDeletable.bulk_delete(objects, False)
 
+    def create_review(self, **kwargs):
+        """ Creates a Review for this label.
+        Kwargs:
+            Review attributes. At a minimum a `Review.score` field
+            value must be provided.
+        """
+        kwargs[Review.label.name] = self
+        kwargs[Review.project.name] = self.project()
+        return self.client._create(Review, kwargs)
+
+
+class Review(DbObject, Deletable, Updateable):
+
+    class NetScore(Enum):
+        Negative = auto()
+        Zero = auto()
+        Positive = auto()
+
+    updated_at = Field.DateTime("updated_at")
+    created_at = Field.DateTime("created_at")
+    score = Field.Float("score")
+
+    created_by = Relationship.ToOne("User", False, "created_by")
+    organization = Relationship.ToOne("Organization", False)
+    project = Relationship.ToOne("Project", False)
+    label = Relationship.ToOne("Label", False)
+
 
 class AssetMetadata(DbObject):
     VIDEO = "VIDEO"

tests/integration/test_review.py

@@ -0,0 +1,78 @@
+import pytest
+
+from labelbox import Review
+from labelbox.exceptions import InvalidQueryError
+
+
+IMG_URL = "https://picsum.photos/200/300"
+
+
+def test_reviews(client, rand_gen):
+    project = client.create_project(name=rand_gen(str))
+    dataset = client.create_dataset(name=rand_gen(str), projects=project)
+    data_row = dataset.create_data_row(row_data=IMG_URL)
+    label = project.create_label(data_row=data_row, label="test",
+                                 seconds_to_label=0.0)
+
+    assert set(label.reviews()) == set()
+    assert set(project.reviews()) == set()
+
+    r1 = label.create_review(score=-1.0)
+    assert r1.project() == project
+    assert r1.label() == label
+    assert r1.score == -1.0
+    assert set(label.reviews()) == {r1}
+    assert set(project.reviews()) == {r1}
+
+    r2 = label.create_review(score=1.0)
+
+    assert set(label.reviews()) == {r1, r2}
+    assert set(project.reviews()) == {r1, r2}
+
+    # Project.reviews supports filtering
+    assert set(project.reviews(where=Review.score > 0.0)) == {r2}
+    assert set(project.reviews(where=Review.score < 0.0)) == {r1}
+
+    # Label.reviews doesn't support filtering
+    with pytest.raises(InvalidQueryError) as exc_info:
+        assert set(label.reviews(where=Review.score > 0.0)) == {r2}
+    assert exc_info.value.message == \
+        "Relationship Label.reviews doesn't support filtering"
+
+    r1.delete()
+
+    assert set(label.reviews()) == {r2}
+    assert set(project.reviews()) == {r2}
+
+    dataset.delete()
+    project.delete()
+
+
+def test_review_metrics(client, rand_gen):
+    project = client.create_project(name=rand_gen(str))
+    dataset = client.create_dataset(name=rand_gen(str), projects=project)
+    data_row = dataset.create_data_row(row_data=IMG_URL)
+    label = project.create_label(data_row=data_row, label="test",
+                                 seconds_to_label=0.0)
+
+    assert project.review_metrics(None) == 1
+    assert project.review_metrics(Review.NetScore.Negative) == 0
+    assert project.review_metrics(Review.NetScore.Zero) == 0
+    assert project.review_metrics(Review.NetScore.Positive) == 0
+
+    for count, score in ((4, 0), (2, 1), (3, -1)):
+        for _ in range(count):
+            l = project.create_label(data_row=data_row, label="l",
+                                     seconds_to_label=0.0)
+            l.create_review(score=score)
+
+    assert project.review_metrics(None) == 1
+    assert project.review_metrics(Review.NetScore.Negative) == 3
+    assert project.review_metrics(Review.NetScore.Zero) == 4
+    assert project.review_metrics(Review.NetScore.Positive) == 2
+
+    with pytest.raises(InvalidQueryError):
+        project.review_metrics(12)
+
+    dataset.delete()
+    project.delete()