Add labeler performance

Author: florijanstamenkovic

labelbox/orm/query.py

@@ -620,3 +620,17 @@ def delete_benchmark(label):
         deleteBenchmark(where: {labelId: $%s}) {id}} """ % (
             label_id_param, label_id_param)
     return query_str, {label_id_param: label.uid}
+
+
+def labeler_performance(project):
+    project_id_param = "projectId"
+    query_str = """query LabelerPerformancePyApi($%s: ID!) {
+        project(where: {id: $%s}) {
+            labelerPerformance(skip: %%d first: %%d) {
+                count user {%s} secondsPerLabel totalTimeLabeling consensus
+                averageBenchmarkAgreement lastActivityTime}
+        }
+    }""" % (project_id_param, project_id_param,
+            results_query_part(Entity.named("User")))
+
+    return query_str, {project_id_param: project.uid}

labelbox/schema.py

@@ -1,3 +1,4 @@
+from collections import namedtuple
 from datetime import datetime, timezone
 from enum import Enum, auto
 import json
@@ -6,6 +7,7 @@
 import os
 import time
 
+from labelbox import utils
 from labelbox.exceptions import InvalidQueryError, ResourceNotFoundError
 from labelbox.orm import query
 from labelbox.orm.db_object import (DbObject, Updateable, Deletable,
@@ -103,6 +105,24 @@ def export_labels(self, timeout_seconds=60):
                          self.uid)
             time.sleep(sleep_time)
 
+    def labeler_performance(self):
+        """ Returns the labeler performances for this Project.
+        Returns:
+            A PaginatedCollection of LabelerPerformance objects.
+        """
+        query_str, params = query.labeler_performance(self)
+
+        def create_labeler_performance(client, result):
+            result["user"] = User(client, result["user"])
+            result["lastActivityTime"] = datetime.fromtimestamp(
+                result["lastActivityTime"] / 1000, timezone.utc)
+            return LabelerPerformance(**{utils.snake_case(key): value
+                                         for key, value in result.items()})
+
+        return PaginatedCollection(self.client, query_str, params,
+                                   ["project", "labelerPerformance"],
+                                   create_labeler_performance)
+
     # TODO Relationships
     # labeledDatasets
     # ...a lot more, define which are required for v0.1
@@ -493,6 +513,13 @@ class User(DbObject):
     # TODO other attributes
 
 
+LabelerPerformance = namedtuple(
+    "LabelerPerformance", "user count seconds_per_label, total_time_labeling "
+    "consensus average_benchmark_agreement last_activity_time")
+LabelerPerformance.__doc__ = "Named tuple containing info about a labeler's " \
+    "performance."
+
+
 class Organization(DbObject):
     """ An Organization is a group of Users associated with data created by
     Users within that Organization. Typically all Users within an Organization

tests/integration/test_labeler_performance.py

@@ -0,0 +1,27 @@
+from datetime import datetime, timezone, timedelta
+import time
+
+
+IMG_URL = "https://picsum.photos/200/300"
+
+
+def test_labeler_performance(client, rand_gen):
+    project = client.create_project(name=rand_gen(str))
+    assert list(project.labeler_performance()) == []
+
+    dataset = client.create_dataset(name=rand_gen(str), projects=project)
+    data_row = dataset.create_data_row(row_data=IMG_URL)
+    label = project.create_label(data_row=data_row, label="test",
+                                 seconds_to_label=0.0)
+    # Sleep a bit as it seems labeler performance isn't updated immediately.
+    time.sleep(5)
+
+    labeler_performance = list(project.labeler_performance())
+    assert len(labeler_performance) == 1
+    my_performance = labeler_performance[0]
+    assert my_performance.user == client.get_user()
+    assert my_performance.count == 1
+    assert isinstance(my_performance.last_activity_time, datetime)
+    now_utc = datetime.now().astimezone(timezone.utc)
+    assert timedelta(0) < now_utc - my_performance.last_activity_time < \
+        timedelta(seconds=30)

tools/db_object_doc_gen.py

@@ -35,13 +35,16 @@
 from labelbox.utils import snake_case
 from labelbox.exceptions import LabelboxError
 from labelbox.orm.db_object import Deletable, BulkDeletable, Updateable
+from labelbox.orm.model import Entity
+import labelbox.schema
 
 
 GENERAL_CLASSES = [labelbox.Client]
 SCHEMA_CLASSES = [
     labelbox.Project, labelbox.Dataset, labelbox.DataRow, labelbox.Label,
     labelbox.AssetMetadata, labelbox.LabelingFrontend, labelbox.Task,
-    labelbox.Webhook, labelbox.User, labelbox.Organization, labelbox.Review]
+    labelbox.Webhook, labelbox.User, labelbox.Organization, labelbox.Review,
+    labelbox.schema.LabelerPerformance]
 
 ERROR_CLASSES = [LabelboxError] + LabelboxError.__subclasses__()
 
@@ -366,16 +369,16 @@ def generate_constants(cls):
     return unordered_list(values)
 
 
-def generate_class(cls, schema_class):
+def generate_class(cls):
     """ Generates HelpDocs style documentation for the given class.
     Args:
         cls (type): The class to generate docs for.
-        schema_class (bool): If `cls` is a DbObject subclass.
     Return:
         HelpDocs style documentation for `cls` containing class description,
         methods and fields and relationships if `schema_class`.
     """
     text = []
+    schema_class = issubclass(cls, Entity)
 
     title = "Class " + cls.__module__ + "." + cls.__name__
     title_id = re.sub(r"\s+", "_", snake_case(title).lower())
@@ -423,11 +426,11 @@ def generate_all():
     text.append(unordered_list([qual_class_name(cls) for cls in ERROR_CLASSES]))
 
     text.append(header(1, "General classes"))
-    text.extend(generate_class(cls, False) for cls in GENERAL_CLASSES)
+    text.extend(generate_class(cls) for cls in GENERAL_CLASSES)
     text.append(header(1, "Data Classes"))
-    text.extend(generate_class(cls, True) for cls in SCHEMA_CLASSES)
+    text.extend(generate_class(cls) for cls in SCHEMA_CLASSES)
     text.append(header(1, "Error Classes"))
-    text.extend(generate_class(cls, False) for cls in ERROR_CLASSES)
+    text.extend(generate_class(cls) for cls in ERROR_CLASSES)
     return "\n".join(text)