Merge pull request #259 from Labelbox/ms/test-fixes

Fix tests

Author: msokoloff1

tests/integration/bulk_import/conftest.py

@@ -6,8 +6,6 @@
 from labelbox.schema.labeling_frontend import LabelingFrontend
 from labelbox.schema.annotation_import import MALPredictionImport
 
-IMG_URL = "https://picsum.photos/200/300"
-
 
 @pytest.fixture
 def ontology():
@@ -103,7 +101,7 @@ def ontology():
 
 
 @pytest.fixture
-def configured_project(client, ontology, rand_gen):
+def configured_project(client, ontology, rand_gen, image_url):
     project = client.create_project(name=rand_gen(str))
     dataset = client.create_dataset(name=rand_gen(str))
     editor = list(
@@ -112,7 +110,7 @@ def configured_project(client, ontology, rand_gen):
     project.setup(editor, ontology)
     data_row_ids = []
     for _ in range(len(ontology['tools']) + len(ontology['classifications'])):
-        data_row_ids.append(dataset.create_data_row(row_data=IMG_URL).uid)
+        data_row_ids.append(dataset.create_data_row(row_data=image_url).uid)
     project.datasets.connect(dataset)
     project.data_row_ids = data_row_ids
     yield project

tests/integration/conftest.py

@@ -10,6 +10,7 @@
 from types import SimpleNamespace
 
 import pytest
+import requests
 
 from labelbox import Client
 from labelbox import LabelingFrontend
@@ -30,7 +31,7 @@ class Environ(Enum):
     STAGING = 'staging'
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def environ() -> Environ:
     """
     Checks environment variables for LABELBOX_ENVIRON to be
@@ -127,11 +128,16 @@ def execute(self, query=None, params=None, check_naming=True, **kwargs):
         return super().execute(query, params, **kwargs)
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def client(environ: str):
     return IntegrationClient(environ)
 
 
+@pytest.fixture(scope="session")
+def image_url(client):
+    return client.upload_data(requests.get(IMG_URL).content, sign=True)
+
+
 @pytest.fixture
 def rand_gen():
 
@@ -187,10 +193,10 @@ def dataset(client, rand_gen):
 
 
 @pytest.fixture
-def datarow(dataset):
+def datarow(dataset, image_url):
     task = dataset.create_data_rows([
         {
-            "row_data": IMG_URL,
+            "row_data": image_url,
             "external_id": "my-image"
         },
     ])
@@ -204,10 +210,10 @@ def datarow(dataset):
 
 
 @pytest.fixture
-def label_pack(project, rand_gen):
+def label_pack(project, rand_gen, image_url):
     client = project.client
     dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
+    data_row = dataset.create_data_row(row_data=image_url)
     label = project.create_label(data_row=data_row, label=rand_gen(str))
     yield LabelPack(project, dataset, data_row, label)
     dataset.delete()
@@ -276,9 +282,9 @@ def project_pack(client):
 
 
 @pytest.fixture
-def configured_project(project, client, rand_gen):
+def configured_project(project, client, rand_gen, image_url):
     dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    dataset.create_data_row(row_data=IMG_URL)
+    dataset.create_data_row(row_data=image_url)
     editor = list(
         project.client.get_labeling_frontends(
             where=LabelingFrontend.name == "editor"))[0]
@@ -318,10 +324,11 @@ def submit(project_id, data_row_id):
 
 
 @pytest.fixture
-def configured_project_with_label(client, rand_gen, annotation_submit_fn):
+def configured_project_with_label(client, rand_gen, annotation_submit_fn,
+                                  image_url):
     project = client.create_project(name=rand_gen(str))
     dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
+    data_row = dataset.create_data_row(row_data=image_url)
     editor = list(
         project.client.get_labeling_frontends(
             where=LabelingFrontend.name == "editor"))[0]

tests/integration/test_data_row_metadata.py

@@ -6,7 +6,6 @@
 from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadata, DeleteDataRowMetadata, \
     DataRowMetadataOntology
 
-IMG_URL = "https://picsum.photos/id/829/200/300"
 FAKE_SCHEMA_ID = "0" * 25
 SPLIT_SCHEMA_ID = "cko8sbczn0002h2dkdaxb5kal"
 TRAIN_SPLIT_ID = "cko8sbscr0003h2dk04w86hof"
@@ -22,13 +21,13 @@ def mdo(client):
 
 
 @pytest.fixture
-def big_dataset(dataset: Dataset):
+def big_dataset(dataset: Dataset, image_url):
     task = dataset.create_data_rows([
         {
-            "row_data": IMG_URL,
+            "row_data": image_url,
             "external_id": "my-image"
         },
-    ] * 1000)
+    ] * 250)
     task.wait_till_done()
 
     yield dataset
@@ -97,35 +96,37 @@ def test_bulk_delete_datarow_metadata(datarow, mdo):
     mdo.bulk_upsert([metadata])
 
     assert len(datarow.metadata["fields"])
-
-    mdo.bulk_delete([
-        DeleteDataRowMetadata(data_row_id=datarow.uid,
-                              fields=[m.schema_id for m in metadata.fields])
-    ])
-
-    assert not len(datarow.metadata["fields"])
+    upload_ids = [m.schema_id for m in metadata.fields]
+    mdo.bulk_delete(
+        [DeleteDataRowMetadata(data_row_id=datarow.uid, fields=upload_ids)])
+    remaining_ids = set([f['schema_id'] for f in datarow.metadata["fields"]])
+    assert not len(remaining_ids.intersection(set(upload_ids)))
 
 
 def test_bulk_partial_delete_datarow_metadata(datarow, mdo):
     """Delete a single from metadata"""
-    assert not len(datarow.metadata["fields"])
+    n_fields = len(datarow.metadata["fields"])
 
     metadata = make_metadata(datarow.uid)
     mdo.bulk_upsert([metadata])
 
-    assert len(datarow.metadata["fields"])
+    assert len(datarow.metadata["fields"]) == (n_fields + 5)
 
     mdo.bulk_delete([
         DeleteDataRowMetadata(data_row_id=datarow.uid, fields=[TEXT_SCHEMA_ID])
     ])
 
-    assert len(datarow.metadata["fields"]) == 4
+    assert len(datarow.metadata["fields"]) == (n_fields + 4)
 
 
 @pytest.mark.slow
 def test_large_bulk_delete_datarow_metadata(big_dataset, mdo):
     metadata = []
-    for dr in big_dataset.export_data_rows():
+    n_fields_start = 0
+    for idx, dr in enumerate(big_dataset.export_data_rows()):
+        if idx == 0:
+            n_fields_start = len(dr.metadata["fields"])
+
         metadata.append(
             DataRowMetadata(data_row_id=dr.uid,
                             fields=[
@@ -151,24 +152,26 @@ def test_large_bulk_delete_datarow_metadata(big_dataset, mdo):
     errors = mdo.bulk_delete(deletes)
     assert len(errors) == 0
     for dr in big_dataset.export_data_rows():
-        assert len(dr.metadata["fields"]) == 1
+        assert len(dr.metadata["fields"]) == 1 + n_fields_start
         break
 
 
 def test_bulk_delete_datarow_enum_metadata(datarow: DataRow, mdo):
     """test bulk deletes for non non fields"""
-    assert not len(datarow.metadata["fields"])
+    n_fields = len(datarow.metadata["fields"])
     metadata = make_metadata(datarow.uid)
     metadata.fields = [
         m for m in metadata.fields if m.schema_id == SPLIT_SCHEMA_ID
     ]
     mdo.bulk_upsert([metadata])
-    assert len(datarow.metadata["fields"])
+    assert len(datarow.metadata["fields"]) == len(
+        set([x.schema_id for x in metadata.fields] +
+            [x['schema_id'] for x in datarow.metadata["fields"]]))
 
     mdo.bulk_delete([
         DeleteDataRowMetadata(data_row_id=datarow.uid, fields=[SPLIT_SCHEMA_ID])
     ])
-    assert not len(datarow.metadata["fields"])
+    assert len(datarow.metadata["fields"]) == n_fields
 
 
 def test_raise_enum_upsert_schema_error(datarow, mdo):
@@ -196,7 +199,6 @@ def test_upsert_non_existent_schema_id(datarow, mdo):
 
 
 def test_delete_non_existent_schema_id(datarow, mdo):
-    assert not len(datarow.metadata["fields"])
     mdo.bulk_delete([
         DeleteDataRowMetadata(data_row_id=datarow.uid,
                               fields=[EMBEDDING_SCHEMA_ID])
@@ -207,15 +209,18 @@ def test_delete_non_existent_schema_id(datarow, mdo):
 @pytest.mark.slow
 def test_large_bulk_delete_non_existent_schema_id(big_dataset, mdo):
     deletes = []
-    for dr in big_dataset.export_data_rows():
+    n_fields_start = 0
+    for idx, dr in enumerate(big_dataset.export_data_rows()):
+        if idx == 0:
+            n_fields_start = len(dr.metadata["fields"])
         deletes.append(
             DeleteDataRowMetadata(data_row_id=dr.uid,
                                   fields=[EMBEDDING_SCHEMA_ID]))
     errors = mdo.bulk_delete(deletes)
     assert len(errors) == 0
 
     for dr in big_dataset.export_data_rows():
-        assert not len(dr.metadata["fields"])
+        assert len(dr.metadata["fields"]) == n_fields_start
         break
 
 

tests/integration/test_data_rows.py

@@ -6,24 +6,22 @@
 from labelbox import DataRow
 from labelbox.exceptions import InvalidQueryError
 
-IMG_URL = "https://picsum.photos/id/829/200/300"
-
 
 def test_get_data_row(datarow, client):
     assert client.get_data_row(datarow.uid)
 
 
-def test_data_row_bulk_creation(dataset, rand_gen):
+def test_data_row_bulk_creation(dataset, rand_gen, image_url):
     client = dataset.client
     assert len(list(dataset.data_rows())) == 0
 
     # Test creation using URL
     task = dataset.create_data_rows([
         {
-            DataRow.row_data: IMG_URL
+            DataRow.row_data: image_url
         },
         {
-            "row_data": IMG_URL
+            "row_data": image_url
         },
     ])
     assert task in client.get_user().created_tasks()
@@ -35,7 +33,7 @@ def test_data_row_bulk_creation(dataset, rand_gen):
 
     data_rows = list(dataset.data_rows())
     assert len(data_rows) == 2
-    assert {data_row.row_data for data_row in data_rows} == {IMG_URL}
+    assert {data_row.row_data for data_row in data_rows} == {image_url}
 
     # Test creation using file name
     with NamedTemporaryFile() as fp:
@@ -48,38 +46,38 @@ def test_data_row_bulk_creation(dataset, rand_gen):
 
     data_rows = list(dataset.data_rows())
     assert len(data_rows) == 3
-    url = ({data_row.row_data for data_row in data_rows} - {IMG_URL}).pop()
+    url = ({data_row.row_data for data_row in data_rows} - {image_url}).pop()
     assert requests.get(url).content == data
 
     data_rows[0].delete()
 
 
 @pytest.mark.slow
-def test_data_row_large_bulk_creation(dataset, rand_gen):
+def test_data_row_large_bulk_creation(dataset, image_url):
     # Do a longer task and expect it not to be complete immediately
     with NamedTemporaryFile() as fp:
         fp.write("Test data".encode())
         fp.flush()
         task = dataset.create_data_rows([{
-            DataRow.row_data: IMG_URL
-        }] * 4500 + [fp.name] * 500)
+            DataRow.row_data: image_url
+        }] * 750 + [fp.name] * 250)
     assert task.status == "IN_PROGRESS"
-    task.wait_till_done()
+    task.wait_till_done(timeout_seconds=120)
     assert task.status == "COMPLETE"
     data_rows = len(list(dataset.data_rows())) == 5003
 
 
 @pytest.mark.xfail(reason="DataRow.dataset() relationship not set")
-def test_data_row_single_creation(dataset, rand_gen):
+def test_data_row_single_creation(dataset, rand_gen, image_url):
     client = dataset.client
     assert len(list(dataset.data_rows())) == 0
 
-    data_row = dataset.create_data_row(row_data=IMG_URL)
+    data_row = dataset.create_data_row(row_data=image_url)
     assert len(list(dataset.data_rows())) == 1
     assert data_row.dataset() == dataset
     assert data_row.created_by() == client.get_user()
     assert data_row.organization() == client.get_organization()
-    assert requests.get(IMG_URL).content == \
+    assert requests.get(image_url).content == \
         requests.get(data_row.row_data).content
     assert data_row.media_attributes is not None
 
@@ -92,9 +90,9 @@ def test_data_row_single_creation(dataset, rand_gen):
         assert requests.get(data_row_2.row_data).content == data
 
 
-def test_data_row_update(dataset, rand_gen):
+def test_data_row_update(dataset, rand_gen, image_url):
     external_id = rand_gen(str)
-    data_row = dataset.create_data_row(row_data=IMG_URL,
+    data_row = dataset.create_data_row(row_data=image_url,
                                        external_id=external_id)
     assert data_row.external_id == external_id
 
@@ -103,14 +101,14 @@ def test_data_row_update(dataset, rand_gen):
     assert data_row.external_id == external_id_2
 
 
-def test_data_row_filtering_sorting(dataset, rand_gen):
+def test_data_row_filtering_sorting(dataset, image_url):
     task = dataset.create_data_rows([
         {
-            DataRow.row_data: IMG_URL,
+            DataRow.row_data: image_url,
             DataRow.external_id: "row1"
         },
         {
-            DataRow.row_data: IMG_URL,
+            DataRow.row_data: image_url,
             DataRow.external_id: "row2"
         },
     ])
@@ -133,9 +131,9 @@ def test_data_row_filtering_sorting(dataset, rand_gen):
         dataset.data_rows(order_by=DataRow.external_id.desc)) == [row2, row1]
 
 
-def test_data_row_deletion(dataset, rand_gen):
+def test_data_row_deletion(dataset, image_url):
     task = dataset.create_data_rows([{
-        DataRow.row_data: IMG_URL,
+        DataRow.row_data: image_url,
         DataRow.external_id: str(i)
     } for i in range(10)])
     task.wait_till_done()
@@ -159,13 +157,13 @@ def test_data_row_deletion(dataset, rand_gen):
     assert {dr.external_id for dr in data_rows} == expected
 
 
-def test_data_row_iteration(dataset, rand_gen) -> None:
+def test_data_row_iteration(dataset, image_url) -> None:
     task = dataset.create_data_rows([
         {
-            DataRow.row_data: IMG_URL
+            DataRow.row_data: image_url
         },
         {
-            "row_data": IMG_URL
+            "row_data": image_url
         },
     ])
     task.wait_till_done()