Merge branch 'develop' of https://github.com/Labelbox/labelbox-python into kkim/polish-tests

Author: Kevin Kim

docs/source/index.rst

@@ -173,14 +173,14 @@ DataRowMetadata
 
 .. automodule:: labelbox.schema.data_row_metadata
    :members:
-   :show-inheritance:   
+   :show-inheritance:
 
 AnnotationImport
 ----------------------------
 
 .. automodule:: labelbox.schema.annotation_import
    :members:
-   :show-inheritance:      
+   :show-inheritance:
 
 Batch
 ----------------------------
@@ -194,4 +194,19 @@ ResourceTag
 
 .. automodule:: labelbox.schema.resource_tag
    :members:
-   :show-inheritance:
+   :show-inheritance:
+
+Slice
+-----------------------------------------
+
+.. automodule:: labelbox.schema.slice
+   :members: Slice
+   :exclude-members: CatalogSlice
+   :show-inheritance:
+
+CatalogSlice
+-----------------------------------------
+.. automodule:: labelbox.schema.slice
+   :members: CatalogSlice
+   :exclude-members: Slice
+   :show-inheritance:

labelbox/__init__.py

@@ -27,3 +27,4 @@
 from labelbox.schema.resource_tag import ResourceTag
 from labelbox.schema.project_resource_tag import ProjectResourceTag
 from labelbox.schema.media_type import MediaType
+from labelbox.schema.slice import Slice, CatalogSlice

labelbox/client.py

@@ -33,6 +33,7 @@
 from labelbox.schema.user import User
 from labelbox.schema.project import Project
 from labelbox.schema.role import Role
+from labelbox.schema.slice import CatalogSlice
 
 from labelbox.schema.media_type import MediaType
 
@@ -963,7 +964,7 @@ def assign_global_keys_to_data_rows(
             timeout_seconds=60) -> Dict[str, Union[str, List[Any]]]:
         """
         Assigns global keys to data rows.
-        
+
         Args:
             A list of dicts containing data_row_id and global_key.
         Returns:
@@ -1211,3 +1212,27 @@ def _format_failed_rows(rows: List[str],
                     "Timed out waiting for get_data_rows_for_global_keys job to complete."
                 )
             time.sleep(sleep_time)
+
+    def get_catalog_slice(self, slice_id) -> CatalogSlice:
+        """
+        Fetches a Catalog Slice by ID.
+
+        Args:
+             slice_id (str): The ID of the Slice
+        Returns:
+            CatalogSlice
+        """
+        query_str = """
+            query getSavedQueryPyApi($id: ID!) {
+                getSavedQuery(id: $id) {
+                    id
+                    name
+                    description
+                    filter
+                    createdAt
+                    updatedAt
+                }
+            }
+        """
+        res = self.execute(query_str, {'id': slice_id})
+        return Entity.CatalogSlice(self, res['getSavedQuery'])

labelbox/data/annotation_types/geometry/mask.py

@@ -54,6 +54,13 @@ def geometry(self) -> Dict[str, Tuple[int, int, int]]:
         external_polygons = self._extract_polygons_from_contours(
             external_contours)
         holes = self._extract_polygons_from_contours(holes)
+
+        if not external_polygons.is_valid:
+            external_polygons = external_polygons.buffer(0)
+
+        if not holes.is_valid:
+            holes = holes.buffer(0)
+
         return external_polygons.difference(holes).__geo_interface__
 
     def draw(self,
@@ -78,7 +85,6 @@ def draw(self,
             np.ndarray representing only this object
                 as opposed to the mask that this object references which might have multiple objects determined by colors
         """
-
         mask = self.mask.value
         mask = np.alltrue(mask == self.color, axis=2).astype(np.uint8)
 

labelbox/data/serialization/coco/instance_dataset.py

@@ -21,7 +21,6 @@ def mask_to_coco_object_annotation(
     # This is going to fill any holes into the multipolygon
     # If you need to support holes use the panoptic data format
     shapely = annotation.value.shapely.simplify(1).buffer(0)
-
     if shapely.is_empty:
         return
 

labelbox/data/serialization/coco/panoptic_dataset.py

@@ -22,7 +22,11 @@ def vector_to_coco_segment_info(canvas: np.ndarray,
                                 annotation: ObjectAnnotation,
                                 annotation_idx: int, image: CocoImage,
                                 category_id: int):
+
     shapely = annotation.value.shapely
+    if shapely.is_empty:
+        return
+
     xmin, ymin, xmax, ymax = shapely.bounds
     canvas = annotation.value.draw(height=image.height,
                                    width=image.width,
@@ -40,6 +44,9 @@ def mask_to_coco_segment_info(canvas: np.ndarray, annotation,
     color = id_to_rgb(annotation_idx)
     mask = annotation.value.draw(color=color)
     shapely = annotation.value.shapely
+    if shapely.is_empty:
+        return
+
     xmin, ymin, xmax, ymax = shapely.bounds
     canvas = np.where(canvas == (0, 0, 0), mask, canvas)
     return SegmentInfo(id=annotation_idx,
@@ -70,20 +77,32 @@ def process_label(label: Label,
         for annotation_idx, annotation in enumerate(annotations[class_name]):
             categories[annotation.name] = hash_category_name(annotation.name)
             if isinstance(annotation.value, Mask):
-                segment, canvas = (mask_to_coco_segment_info(
+                coco_segment_info = mask_to_coco_segment_info(
                     canvas, annotation, class_idx + 1,
-                    categories[annotation.name]))
+                    categories[annotation.name])
+
+                if coco_segment_info is None:
+                    # Filter out empty masks
+                    continue
+
+                segment, canvas = coco_segment_info
                 segments.append(segment)
                 is_thing[annotation.name] = 0
 
             elif isinstance(annotation.value, (Polygon, Rectangle)):
-                segment, canvas = vector_to_coco_segment_info(
+                coco_vector_info = vector_to_coco_segment_info(
                     canvas,
                     annotation,
                     annotation_idx=(class_idx if all_stuff else annotation_idx)
                     + 1,
                     image=image,
                     category_id=categories[annotation.name])
+
+                if coco_segment_info is None:
+                    # Filter out empty annotations
+                    continue
+
+                segment, canvas = coco_vector_info
                 segments.append(segment)
                 is_thing[annotation.name] = 1 - int(all_stuff)
 

labelbox/orm/model.py

@@ -377,6 +377,7 @@ class Entity(metaclass=EntityMeta):
         ProjectRole: Type[labelbox.ProjectRole]
         Project: Type[labelbox.Project]
         Batch: Type[labelbox.Batch]
+        CatalogSlice: Type[labelbox.CatalogSlice]
 
     @classmethod
     def _attributes_of_type(cls, attr_type):

labelbox/schema/slice.py

@@ -0,0 +1,59 @@
+from labelbox.orm.db_object import DbObject
+from labelbox.orm.model import Field
+from labelbox.pagination import PaginatedCollection
+
+
+class Slice(DbObject):
+    """
+    A Slice is a saved set of filters (saved query).
+    This is an abstract class and should not be instantiated.
+
+    Attributes:
+        name (datetime)
+        description (datetime)
+        created_at (datetime)
+        updated_at (datetime)
+        filter (json)
+    """
+    name = Field.String("name")
+    description = Field.String("description")
+    created_at = Field.DateTime("created_at")
+    updated_at = Field.DateTime("updated_at")
+    filter = Field.Json("filter")
+
+
+class CatalogSlice(Slice):
+    """
+    Represents a Slice used for filtering data rows in Catalog.
+    """
+
+    def get_data_row_ids(self) -> PaginatedCollection:
+        """
+        Fetches all data row ids that match this Slice
+
+        Returns:
+            A PaginatedCollection of data row ids
+        """
+        query_str = """
+            query getDataRowIdsBySavedQueryPyApi($id: ID!, $from: String, $first: Int!) {
+                getDataRowIdsBySavedQuery(input: {
+                    savedQueryId: $id,
+                    after: $from
+                    first: $first
+                }) {
+                    totalCount
+                    nodes
+                    pageInfo {
+                        endCursor
+                        hasNextPage
+                    }
+                }
+            }
+        """
+        return PaginatedCollection(
+            client=self.client,
+            query=query_str,
+            params={'id': self.uid},
+            dereferencing=['getDataRowIdsBySavedQuery', 'nodes'],
+            obj_class=lambda _, data_row_id: data_row_id,
+            cursor_path=['getDataRowIdsBySavedQuery', 'pageInfo', 'endCursor'])

tests/integration/test_data_rows.py

@@ -352,7 +352,7 @@ def test_create_data_rows_with_invalid_metadata(dataset, image_url):
         DataRow.metadata_fields: fields
     }])
     task.wait_till_done()
-    assert task.status == "COMPLETE"
+    assert task.status == "FAILED"
     assert len(task.failed_data_rows) > 0
 
 
@@ -633,9 +633,10 @@ def test_data_row_bulk_creation_with_same_global_keys(dataset, sample_image):
     }])
 
     task.wait_till_done()
-    assert task.status == "COMPLETE"
+    assert task.status == "FAILED"
     assert len(task.failed_data_rows) > 0
     assert len(list(dataset.data_rows())) == 0
+    assert task.errors == "Import job failed"
 
     task = dataset.create_data_rows([{
         DataRow.row_data: sample_image,

tests/integration/test_task.py

@@ -1,6 +1,6 @@
 import pytest
 
-from labelbox import DataRow, Task
+from labelbox import DataRow
 from labelbox.schema.data_row_metadata import DataRowMetadataField
 
 EMBEDDING_SCHEMA_ID = "ckpyije740000yxdk81pbgjdc"
@@ -21,13 +21,12 @@ def test_task_errors(dataset, image_url):
             ]
         },
     ])
+
     assert task in client.get_user().created_tasks()
     task.wait_till_done()
-    assert task.status == "COMPLETE"
+    assert task.status == "FAILED"
     assert len(task.failed_data_rows) > 0
     assert task.errors is not None
-    assert 'message' in task.errors[0]
-    assert len(task.result) == 0
 
 
 def test_task_success_json(dataset, image_url):