Merge branch 'develop' of https://github.com/Labelbox/labelbox-python into ms/attachments

Author: Matt Sokoloff

labelbox/data/serialization/labelbox_v1/label.py

@@ -131,7 +131,7 @@ class LBV1Label(BaseModel):
     seconds_to_label: Optional[float] = Extra('Seconds to Label')
     agreement: Optional[float] = Extra('Agreement')
     benchmark_agreement: Optional[float] = Extra('Benchmark Agreement')
-    benchmark_id: Optional[float] = Extra('Benchmark ID')
+    benchmark_id: Optional[str] = Extra('Benchmark ID')
     dataset_name: Optional[str] = Extra('Dataset Name')
     reviews: Optional[List[Review]] = Extra('Reviews')
     label_url: Optional[str] = Extra('View Label')

labelbox/schema/project.py

@@ -582,6 +582,26 @@ def enable_model_assisted_labeling(self, toggle: bool = True) -> bool:
         return res["project"]["showPredictionsToLabelers"][
             "showingPredictionsToLabelers"]
 
+    def bulk_import_requests(self):
+        """ Returns bulk import request objects which are used in model-assisted labeling.
+        These are returned with the oldest first, and most recent last.
+        """
+
+        id_param = "project_id"
+        query_str = """query ListAllImportRequestsPyApi($%s: ID!) {
+            bulkImportRequests (
+                where: { projectId: $%s }
+                skip: %%d
+                first: %%d
+            ) {
+                %s
+            }
+        }""" % (id_param, id_param,
+                query.results_query_part(Entity.BulkImportRequest))
+        return PaginatedCollection(self.client, query_str,
+                                   {id_param: str(self.uid)},
+                                   ["bulkImportRequests"], BulkImportRequest)
+
     def upload_annotations(
             self,
             name: str,

tests/integration/bulk_import/conftest.py

@@ -6,8 +6,6 @@
 from labelbox.schema.labeling_frontend import LabelingFrontend
 from labelbox.schema.annotation_import import MALPredictionImport
 
-IMG_URL = "https://picsum.photos/200/300"
-
 
 @pytest.fixture
 def ontology():
@@ -103,7 +101,7 @@ def ontology():
 
 
 @pytest.fixture
-def configured_project(client, ontology, rand_gen):
+def configured_project(client, ontology, rand_gen, image_url):
     project = client.create_project(name=rand_gen(str))
     dataset = client.create_dataset(name=rand_gen(str))
     editor = list(
@@ -112,7 +110,7 @@ def configured_project(client, ontology, rand_gen):
     project.setup(editor, ontology)
     data_row_ids = []
     for _ in range(len(ontology['tools']) + len(ontology['classifications'])):
-        data_row_ids.append(dataset.create_data_row(row_data=IMG_URL).uid)
+        data_row_ids.append(dataset.create_data_row(row_data=image_url).uid)
     project.datasets.connect(dataset)
     project.data_row_ids = data_row_ids
     yield project

tests/integration/bulk_import/test_bulk_import_request.py

@@ -149,21 +149,38 @@ def assert_file_content(url: str, predictions):
     assert response.text == ndjson.dumps(predictions)
 
 
-def test_delete(client, configured_project, predictions):
+def test_project_bulk_import_requests(client, configured_project, predictions):
+    result = configured_project.bulk_import_requests()
+    assert len(list(result)) == 0
+
+    name = str(uuid.uuid4())
+    bulk_import_request = configured_project.upload_annotations(
+        name=name, annotations=predictions)
+    bulk_import_request.wait_until_done()
 
-    id_param = "project_id"
-    query_str = """query bulk_import_requestsPyApi($%s: ID!) {bulkImportRequests(where: {projectId: $%s}) {id}}""" % (
-        id_param, id_param)
+    name = str(uuid.uuid4())
+    bulk_import_request = configured_project.upload_annotations(
+        name=name, annotations=predictions)
+    bulk_import_request.wait_until_done()
+
+    name = str(uuid.uuid4())
+    bulk_import_request = configured_project.upload_annotations(
+        name=name, annotations=predictions)
+    bulk_import_request.wait_until_done()
+
+    result = configured_project.bulk_import_requests()
+    assert len(list(result)) == 3
+
+
+def test_delete(client, configured_project, predictions):
     name = str(uuid.uuid4())
 
     bulk_import_request = configured_project.upload_annotations(
         name=name, annotations=predictions)
     bulk_import_request.wait_until_done()
-    all_import_requests = client.execute(query_str,
-                                         {id_param: configured_project.uid})
-    assert len(all_import_requests['bulkImportRequests']) == 1
+    all_import_requests = configured_project.bulk_import_requests()
+    assert len(list(all_import_requests)) == 1
 
     bulk_import_request.delete()
-    all_import_requests = client.execute(query_str,
-                                         {id_param: configured_project.uid})
-    assert len(all_import_requests['bulkImportRequests']) == 0
+    all_import_requests = configured_project.bulk_import_requests()
+    assert len(list(all_import_requests)) == 0

tests/integration/conftest.py

@@ -10,6 +10,7 @@
 from types import SimpleNamespace
 
 import pytest
+import requests
 
 from labelbox import Client
 from labelbox import LabelingFrontend
@@ -30,7 +31,7 @@ class Environ(Enum):
     STAGING = 'staging'
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def environ() -> Environ:
     """
     Checks environment variables for LABELBOX_ENVIRON to be
@@ -127,11 +128,16 @@ def execute(self, query=None, params=None, check_naming=True, **kwargs):
         return super().execute(query, params, **kwargs)
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def client(environ: str):
     return IntegrationClient(environ)
 
 
+@pytest.fixture(scope="session")
+def image_url(client):
+    return client.upload_data(requests.get(IMG_URL).content, sign=True)
+
+
 @pytest.fixture
 def rand_gen():
 
@@ -187,10 +193,10 @@ def dataset(client, rand_gen):
 
 
 @pytest.fixture
-def datarow(dataset):
+def datarow(dataset, image_url):
     task = dataset.create_data_rows([
         {
-            "row_data": IMG_URL,
+            "row_data": image_url,
             "external_id": "my-image"
         },
     ])
@@ -204,10 +210,10 @@ def datarow(dataset):
 
 
 @pytest.fixture
-def label_pack(project, rand_gen):
+def label_pack(project, rand_gen, image_url):
     client = project.client
     dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
+    data_row = dataset.create_data_row(row_data=image_url)
     label = project.create_label(data_row=data_row, label=rand_gen(str))
     yield LabelPack(project, dataset, data_row, label)
     dataset.delete()
@@ -276,9 +282,9 @@ def project_pack(client):
 
 
 @pytest.fixture
-def configured_project(project, client, rand_gen):
+def configured_project(project, client, rand_gen, image_url):
     dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    dataset.create_data_row(row_data=IMG_URL)
+    dataset.create_data_row(row_data=image_url)
     editor = list(
         project.client.get_labeling_frontends(
             where=LabelingFrontend.name == "editor"))[0]
@@ -318,10 +324,11 @@ def submit(project_id, data_row_id):
 
 
 @pytest.fixture
-def configured_project_with_label(client, rand_gen, annotation_submit_fn):
+def configured_project_with_label(client, rand_gen, annotation_submit_fn,
+                                  image_url):
     project = client.create_project(name=rand_gen(str))
     dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
+    data_row = dataset.create_data_row(row_data=image_url)
     editor = list(
         project.client.get_labeling_frontends(
             where=LabelingFrontend.name == "editor"))[0]

tests/integration/test_data_row_metadata.py

@@ -6,7 +6,6 @@
 from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadata, DeleteDataRowMetadata, \
     DataRowMetadataOntology
 
-IMG_URL = "https://picsum.photos/id/829/200/300"
 FAKE_SCHEMA_ID = "0" * 25
 SPLIT_SCHEMA_ID = "cko8sbczn0002h2dkdaxb5kal"
 TRAIN_SPLIT_ID = "cko8sbscr0003h2dk04w86hof"
@@ -22,13 +21,13 @@ def mdo(client):
 
 
 @pytest.fixture
-def big_dataset(dataset: Dataset):
+def big_dataset(dataset: Dataset, image_url):
     task = dataset.create_data_rows([
         {
-            "row_data": IMG_URL,
+            "row_data": image_url,
             "external_id": "my-image"
         },
-    ] * 1000)
+    ] * 250)
     task.wait_till_done()
 
     yield dataset
@@ -97,35 +96,37 @@ def test_bulk_delete_datarow_metadata(datarow, mdo):
     mdo.bulk_upsert([metadata])
 
     assert len(datarow.metadata["fields"])
-
-    mdo.bulk_delete([
-        DeleteDataRowMetadata(data_row_id=datarow.uid,
-                              fields=[m.schema_id for m in metadata.fields])
-    ])
-
-    assert not len(datarow.metadata["fields"])
+    upload_ids = [m.schema_id for m in metadata.fields]
+    mdo.bulk_delete(
+        [DeleteDataRowMetadata(data_row_id=datarow.uid, fields=upload_ids)])
+    remaining_ids = set([f['schema_id'] for f in datarow.metadata["fields"]])
+    assert not len(remaining_ids.intersection(set(upload_ids)))
 
 
 def test_bulk_partial_delete_datarow_metadata(datarow, mdo):
     """Delete a single from metadata"""
-    assert not len(datarow.metadata["fields"])
+    n_fields = len(datarow.metadata["fields"])
 
     metadata = make_metadata(datarow.uid)
     mdo.bulk_upsert([metadata])
 
-    assert len(datarow.metadata["fields"])
+    assert len(datarow.metadata["fields"]) == (n_fields + 5)
 
     mdo.bulk_delete([
         DeleteDataRowMetadata(data_row_id=datarow.uid, fields=[TEXT_SCHEMA_ID])
     ])
 
-    assert len(datarow.metadata["fields"]) == 4
+    assert len(datarow.metadata["fields"]) == (n_fields + 4)
 
 
 @pytest.mark.slow
 def test_large_bulk_delete_datarow_metadata(big_dataset, mdo):
     metadata = []
-    for dr in big_dataset.export_data_rows():
+    n_fields_start = 0
+    for idx, dr in enumerate(big_dataset.export_data_rows()):
+        if idx == 0:
+            n_fields_start = len(dr.metadata["fields"])
+
         metadata.append(
             DataRowMetadata(data_row_id=dr.uid,
                             fields=[
@@ -151,24 +152,26 @@ def test_large_bulk_delete_datarow_metadata(big_dataset, mdo):
     errors = mdo.bulk_delete(deletes)
     assert len(errors) == 0
     for dr in big_dataset.export_data_rows():
-        assert len(dr.metadata["fields"]) == 1
+        assert len(dr.metadata["fields"]) == 1 + n_fields_start
         break
 
 
 def test_bulk_delete_datarow_enum_metadata(datarow: DataRow, mdo):
     """test bulk deletes for non non fields"""
-    assert not len(datarow.metadata["fields"])
+    n_fields = len(datarow.metadata["fields"])
     metadata = make_metadata(datarow.uid)
     metadata.fields = [
         m for m in metadata.fields if m.schema_id == SPLIT_SCHEMA_ID
     ]
     mdo.bulk_upsert([metadata])
-    assert len(datarow.metadata["fields"])
+    assert len(datarow.metadata["fields"]) == len(
+        set([x.schema_id for x in metadata.fields] +
+            [x['schema_id'] for x in datarow.metadata["fields"]]))
 
     mdo.bulk_delete([
         DeleteDataRowMetadata(data_row_id=datarow.uid, fields=[SPLIT_SCHEMA_ID])
     ])
-    assert not len(datarow.metadata["fields"])
+    assert len(datarow.metadata["fields"]) == n_fields
 
 
 def test_raise_enum_upsert_schema_error(datarow, mdo):
@@ -196,7 +199,6 @@ def test_upsert_non_existent_schema_id(datarow, mdo):
 
 
 def test_delete_non_existent_schema_id(datarow, mdo):
-    assert not len(datarow.metadata["fields"])
     mdo.bulk_delete([
         DeleteDataRowMetadata(data_row_id=datarow.uid,
                               fields=[EMBEDDING_SCHEMA_ID])
@@ -207,15 +209,18 @@ def test_delete_non_existent_schema_id(datarow, mdo):
 @pytest.mark.slow
 def test_large_bulk_delete_non_existent_schema_id(big_dataset, mdo):
     deletes = []
-    for dr in big_dataset.export_data_rows():
+    n_fields_start = 0
+    for idx, dr in enumerate(big_dataset.export_data_rows()):
+        if idx == 0:
+            n_fields_start = len(dr.metadata["fields"])
         deletes.append(
             DeleteDataRowMetadata(data_row_id=dr.uid,
                                   fields=[EMBEDDING_SCHEMA_ID]))
     errors = mdo.bulk_delete(deletes)
     assert len(errors) == 0
 
     for dr in big_dataset.export_data_rows():
-        assert not len(dr.metadata["fields"])
+        assert len(dr.metadata["fields"]) == n_fields_start
         break