Merge branch 'develop' of https://github.com/Labelbox/labelbox-python into kkim/AL-2219

Author: Kevin Kim

CHANGELOG.md

@@ -1,5 +1,14 @@
 # Changelog
 
+# Version 3.21.0
+## Added
+  * Projects can be created with a `media_type`
+  * Added `media_type` attribute to `Project`
+  * New `MediaType` enumeration
+
+## Fix
+  * Added back the mimetype to datarow bulk uploads for orgs that require delegated access
+
 # Version 3.20.1 (2022-05-02)
 ## Updated
 * Ontology Classification `scope` field is only set for top level classifications

labelbox/__init__.py

@@ -1,11 +1,11 @@
 name = "labelbox"
-__version__ = "3.20.1"
+__version__ = "3.21.0"
 
 import sys
 import warnings
 
 if sys.version_info < (3, 7):
-    warnings.warn("""Python 3.6 will no longer be actively supported 
+    warnings.warn("""Python 3.6 will no longer be actively supported
     starting 06/01/2022. Please upgrade to Python 3.7 or higher.""")
 
 from labelbox.client import Client

labelbox/client.py

@@ -90,6 +90,7 @@ def __init__(self,
             'Authorization': 'Bearer %s' % api_key,
             'X-User-Agent': f'python-sdk {SDK_VERSION}'
         }
+        self._data_row_metadata_ontology = None
 
     @retry.Retry(predicate=retry.if_exception_type(
         labelbox.exceptions.InternalServerError))
@@ -648,7 +649,9 @@ def get_data_row_metadata_ontology(self) -> DataRowMetadataOntology:
             DataRowMetadataOntology: The ontology for Data Row Metadata for an organization
 
         """
-        return DataRowMetadataOntology(self)
+        if self._data_row_metadata_ontology is None:
+            self._data_row_metadata_ontology = DataRowMetadataOntology(self)
+        return self._data_row_metadata_ontology
 
     def get_model(self, model_id) -> Model:
         """ Gets a single Model with the given ID.

labelbox/exceptions.py

@@ -71,6 +71,12 @@ class InvalidQueryError(LabelboxError):
     pass
 
 
+class ResourceCreationError(LabelboxError):
+    """ Indicates that a resource could not be created in the server side
+    due to a validation or transaction error"""
+    pass
+
+
 class NetworkError(LabelboxError):
     """Raised when an HTTPError occurs."""
 
@@ -122,4 +128,4 @@ class MALValidationError(LabelboxError):
 
 class OperationNotAllowedException(Exception):
     """Raised when user does not have permissions to a resource or has exceeded usage limit"""
-    pass
+    pass

labelbox/orm/model.py

@@ -32,8 +32,9 @@ class Field:
     Attributes:
         field_type (Field.Type): The type of the field.
         name (str): name that the attribute has in client-side Python objects
-        grapgql_name (str): name that the attribute has in queries (and in
+        graphql_name (str): name that the attribute has in queries (and in
             server-side database definition).
+        result_subquery (str): graphql query result payload for a field.
     """
 
     class Type(Enum):
@@ -55,13 +56,23 @@ def name(self):
             return self.enum_cls.__name__
 
     class ListType:
+        """ Represents Field that is a list of some object.
+        Args:
+            list_cls (type): Type of object that list is made of.
+            graphql_type (str): Inner object's graphql type.
+                By default, the list_cls's name is used as the graphql type.
+        """
 
-        def __init__(self, list_cls: type):
+        def __init__(self, list_cls: type, graphql_type=None):
             self.list_cls = list_cls
+            if graphql_type is None:
+                self.graphql_type = self.list_cls.__name__
+            else:
+                self.graphql_type = graphql_type
 
         @property
         def name(self):
-            return self.list_cls.__name__
+            return f"[{self.graphql_type}]"
 
     class Order(Enum):
         """ Type of sort ordering. """
@@ -101,13 +112,14 @@ def Json(*args):
         return Field(Field.Type.Json, *args)
 
     @staticmethod
-    def List(list_cls: type, *args):
-        return Field(Field.ListType(list_cls), *args)
+    def List(list_cls: type, graphql_type=None, **kwargs):
+        return Field(Field.ListType(list_cls, graphql_type), **kwargs)
 
     def __init__(self,
                  field_type: Union[Type, EnumType, ListType],
                  name,
-                 graphql_name=None):
+                 graphql_name=None,
+                 result_subquery=None):
         """ Field init.
         Args:
             field_type (Field.Type): The type of the field.
@@ -116,12 +128,14 @@ def __init__(self,
             graphql_name (str): query and server-side name of a database object.
                 If None, it is constructed from the client-side name by converting
                 snake_case (Python convention) into camelCase (GraphQL convention).
+            result_subquery (str): graphql query result payload for a field.
         """
         self.field_type = field_type
         self.name = name
         if graphql_name is None:
             graphql_name = utils.camel_case(name)
         self.graphql_name = graphql_name
+        self.result_subquery = result_subquery
 
     @property
     def asc(self):

labelbox/orm/query.py

@@ -42,8 +42,9 @@ def results_query_part(entity):
     """
     # Query for fields
     fields = [
-        field.graphql_name if field.graphql_name != "customMetadata" else
-        "customMetadata { value schemaId }" for field in entity.fields()
+        field.result_subquery
+        if field.result_subquery is not None else field.graphql_name
+        for field in entity.fields()
     ]
 
     # Query for cached relationships

labelbox/schema/batch.py

@@ -104,8 +104,10 @@ def export_data_rows(self, timeout_seconds=120) -> Generator:
                 response = requests.get(download_url)
                 response.raise_for_status()
                 reader = ndjson.reader(StringIO(response.text))
-                return (
-                    Entity.DataRow(self.client, result) for result in reader)
+                # TODO: Update result to parse customMetadata when resolver returns
+                return (Entity.DataRow(self.client, {
+                    **result, 'customMetadata': []
+                }) for result in reader)
             elif res["status"] == "FAILED":
                 raise LabelboxError("Data row export failed.")
 

labelbox/schema/data_row.py

@@ -1,11 +1,10 @@
 import logging
-from datetime import datetime
-from typing import List, Dict, Union, TYPE_CHECKING
+from typing import TYPE_CHECKING
 
 from labelbox.orm import query
 from labelbox.orm.db_object import DbObject, Updateable, BulkDeletable
 from labelbox.orm.model import Entity, Field, Relationship
-from labelbox.schema.data_row_metadata import DataRowMetadataField
+from labelbox.schema.data_row_metadata import DataRowMetadataField  # type: ignore
 
 if TYPE_CHECKING:
     from labelbox import AssetAttachment
@@ -23,6 +22,7 @@ class DataRow(DbObject, Updateable, BulkDeletable):
         updated_at (datetime)
         created_at (datetime)
         media_attributes (dict): generated media attributes for the datarow
+        custom_metadata (list): metadata associated with the datarow
 
         dataset (Relationship): `ToOne` relationship to Dataset
         created_by (Relationship): `ToOne` relationship to User
@@ -35,7 +35,11 @@ class DataRow(DbObject, Updateable, BulkDeletable):
     updated_at = Field.DateTime("updated_at")
     created_at = Field.DateTime("created_at")
     media_attributes = Field.Json("media_attributes")
-    custom_metadata = Field.List(DataRowMetadataField, "custom_metadata")
+    custom_metadata = Field.List(
+        DataRowMetadataField,
+        graphql_type="DataRowCustomMetadataUpsertInput!",
+        name="custom_metadata",
+        result_subquery="customMetadata { value schemaId }")
 
     # Relationships
     dataset = Relationship.ToOne("Dataset")
@@ -97,4 +101,4 @@ def create_attachment(self, attachment_type,
                 data_row_id_param: self.uid
             })
         return Entity.AssetAttachment(self.client,
-                                      res["createDataRowAttachment"])
+                                      res["createDataRowAttachment"])

labelbox/schema/data_row_metadata.py

@@ -200,6 +200,10 @@ def _parse_ontology(raw_ontology) -> List[DataRowMetadataSchema]:
 
         return fields
 
+    def refresh_ontology(self):
+        self._raw_ontology = self._get_ontology()
+        self._build_ontology()
+
     def parse_metadata(
         self, unparsed: List[Dict[str,
                                   List[Union[str,
@@ -221,6 +225,14 @@ def parse_metadata(
         for dr in unparsed:
             fields = []
             for f in dr["fields"]:
+                if f["schemaId"] not in self.fields_by_id:
+                    # Update metadata ontology if field can't be found
+                    self.refresh_ontology()
+                    if f["schemaId"] not in self.fields_by_id:
+                        raise ValueError(
+                            f"Schema Id `{f['schemaId']}` not found in ontology"
+                        )
+
                 schema = self.fields_by_id[f["schemaId"]]
                 if schema.kind == DataRowMetadataKind.enum:
                     continue
@@ -297,7 +309,6 @@ def _batch_upsert(
                         chain.from_iterable(
                             self.parse_upsert(m) for m in m.fields))).dict(
                                 by_alias=True))
-
         res = _batch_operations(_batch_upsert, items, self._batch_size)
         return res
 
@@ -399,8 +410,11 @@ def parse_upsert(
         """Format for metadata upserts to GQL"""
 
         if metadatum.schema_id not in self.fields_by_id:
-            raise ValueError(
-                f"Schema Id `{metadatum.schema_id}` not found in ontology")
+            # Update metadata ontology if field can't be found
+            self.refresh_ontology()
+            if metadatum.schema_id not in self.fields_by_id:
+                raise ValueError(
+                    f"Schema Id `{metadatum.schema_id}` not found in ontology")
 
         schema = self.fields_by_id[metadatum.schema_id]
 
@@ -428,8 +442,11 @@ def _validate_delete(self, delete: DeleteDataRowMetadata):
         deletes = set()
         for schema_id in delete.fields:
             if schema_id not in self.fields_by_id:
-                raise ValueError(
-                    f"Schema Id `{schema_id}` not found in ontology")
+                # Update metadata ontology if field can't be found
+                self.refresh_ontology()
+                if schema_id not in self.fields_by_id:
+                    raise ValueError(
+                        f"Schema Id `{schema_id}` not found in ontology")
 
             schema = self.fields_by_id[schema_id]
             # handle users specifying enums by adding all option enums

labelbox/schema/dataset.py

@@ -79,6 +79,18 @@ def create_data_row(self, **kwargs) -> "DataRow":
         if os.path.exists(row_data):
             kwargs[DataRow.row_data.name] = self.client.upload_file(row_data)
         kwargs[DataRow.dataset.name] = self
+
+        # Parse metadata fields, if they are provided
+        if DataRow.custom_metadata.name in kwargs:
+            mdo = self.client.get_data_row_metadata_ontology()
+            metadata_fields = kwargs[DataRow.custom_metadata.name]
+            metadata = list(
+                chain.from_iterable(
+                    mdo.parse_upsert(m) for m in metadata_fields))
+            kwargs[DataRow.custom_metadata.name] = [
+                md.dict(by_alias=True) for md in metadata
+            ]
+
         return self.client._create(DataRow, kwargs)
 
     def create_data_rows_sync(self, items) -> None:
@@ -344,8 +356,9 @@ def convert_item(item):
             items = [future.result() for future in as_completed(futures)]
         # Prepare and upload the desciptor file
         data = json.dumps(items)
-        #print(f"!! input data: {data}")
-        return self.client.upload_data(data)
+        return self.client.upload_data(data,
+                                       content_type="application/json",
+                                       filename="json_import.json")
 
     def data_rows_for_external_id(self,
                                   external_id,
@@ -426,8 +439,10 @@ def export_data_rows(self, timeout_seconds=120) -> Generator:
                 response = requests.get(download_url)
                 response.raise_for_status()
                 reader = ndjson.reader(StringIO(response.text))
-                return (
-                    Entity.DataRow(self.client, result) for result in reader)
+                # TODO: Update result to parse customMetadata when resolver returns
+                return (Entity.DataRow(self.client, {
+                    **result, 'customMetadata': []
+                }) for result in reader)
             elif res["status"] == "FAILED":
                 raise LabelboxError("Data row export failed.")