Improve tests.

Author: florijanstamenkovic

tests/integration/conftest.py

@@ -1,3 +1,4 @@
+from collections import namedtuple
 from datetime import datetime
 import os
 from random import randint
@@ -7,7 +8,9 @@
 import pytest
 
 from labelbox import Client
-from labelbox.orm.model import Field
+
+
+IMG_URL = "https://picsum.photos/200/300"
 
 
 class IntegrationClient(Client):
@@ -34,14 +37,42 @@ def client():
 @pytest.fixture
 def rand_gen():
     def gen(field_type):
-        if field_type == str:
+        if field_type is str:
             return "".join(ascii_letters[randint(0, len(ascii_letters) - 1)]
                             for _ in range(16))
 
-        if field_type == datetime:
+        if field_type is datetime:
             return datetime.now()
 
         raise Exception("Can't random generate for field type '%r'" %
                         field.field_type)
 
     return gen
+
+
+@pytest.fixture
+def project(client, rand_gen):
+    project = client.create_project(name=rand_gen(str))
+    yield project
+    project.delete()
+
+
+@pytest.fixture
+def dataset(client, rand_gen):
+    dataset = client.create_dataset(name=rand_gen(str))
+    yield dataset
+    dataset.delete()
+
+
+LabelPack = namedtuple("LabelPack", "project dataset data_row label")
+
+
+@pytest.fixture
+def label_pack(project, rand_gen):
+    client = project.client
+    dataset = client.create_dataset(name=rand_gen(str), projects=project)
+    data_row = dataset.create_data_row(row_data=IMG_URL)
+    label = project.create_label(data_row=data_row, label=rand_gen(str),
+                                 seconds_to_label=0.0)
+    yield LabelPack(project, dataset, data_row, label)
+    dataset.delete()

tests/integration/test_asset_metadata.py

@@ -7,8 +7,7 @@
 IMG_URL = "https://picsum.photos/200/300"
 
 
-def test_asset_metadata_crud(client, rand_gen):
-    dataset = client.create_dataset(name=rand_gen(str))
+def test_asset_metadata_crud(dataset, rand_gen):
     data_row = dataset.create_data_row(row_data=IMG_URL)
     assert len(list(data_row.metadata())) == 0
 
@@ -27,5 +26,3 @@ def test_asset_metadata_crud(client, rand_gen):
         data_row.metadata(order_by=AssetMetadata.meta_value.asc)
     assert exc_info.value.message == \
         "Relationship DataRow.metadata doesn't support sorting"
-
-    dataset.delete()

tests/integration/test_benchmark.py

@@ -1,12 +1,5 @@
-IMG_URL = "https://picsum.photos/200/300"
-
-
-def test_benchmark(client, rand_gen):
-    project = client.create_project(name=rand_gen(str))
-    dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
-    label = project.create_label(data_row=data_row, label="test",
-                                 seconds_to_label=0.0)
+def test_benchmark(label_pack):
+    project, _, data_row, label = label_pack
     assert set(project.benchmarks()) == set()
     assert label.is_benchmark_reference == False
 
@@ -22,6 +15,3 @@ def test_benchmark(client, rand_gen):
     # Refresh label data to check it's benchmark reference
     label = list(data_row.labels())[0]
     assert label.is_benchmark_reference == False
-
-    dataset.delete()
-    project.delete()

tests/integration/test_client_errors.py

@@ -9,7 +9,7 @@
 import labelbox.exceptions
 
 
-def test_missing_api_key(client, rand_gen):
+def test_missing_api_key(client):
     key = os.environ.get(labelbox.client._LABELBOX_API_KEY, None)
     if key is not None:
         del os.environ[labelbox.client._LABELBOX_API_KEY]

tests/integration/test_data_rows.py

@@ -3,16 +3,15 @@
 import pytest
 import requests
 
-from labelbox import Project, Dataset, DataRow
+from labelbox import DataRow
 from labelbox.exceptions import InvalidQueryError
 
 
 IMG_URL = "https://picsum.photos/200/300"
 
 
-def test_data_row_bulk_creation(client, rand_gen):
-    dataset = client.create_dataset(name=rand_gen(str))
-
+def test_data_row_bulk_creation(dataset, rand_gen):
+    client = dataset.client
     assert len(list(dataset.data_rows())) == 0
 
     # Test creation using URL
@@ -59,11 +58,9 @@ def test_data_row_bulk_creation(client, rand_gen):
     assert task.status == "COMPLETE"
     data_rows = len(list(dataset.data_rows())) == 5003
 
-    dataset.delete()
-
 
-def test_data_row_single_creation(client, rand_gen):
-    dataset = client.create_dataset(name=rand_gen(str))
+def test_data_row_single_creation(dataset, rand_gen):
+    client = dataset.client
     assert len(list(dataset.data_rows())) == 0
 
     data_row = dataset.create_data_row(row_data=IMG_URL)
@@ -77,11 +74,8 @@ def test_data_row_single_creation(client, rand_gen):
         data_row_2 = dataset.create_data_row(row_data=fp.name)
         assert len(list(dataset.data_rows())) == 2
 
-    dataset.delete()
-
 
-def test_data_row_update(client, rand_gen):
-    dataset = client.create_dataset(name=rand_gen(str))
+def test_data_row_update(dataset, rand_gen):
     external_id = rand_gen(str)
     data_row = dataset.create_data_row(row_data=IMG_URL, external_id=external_id)
     assert data_row.external_id == external_id
@@ -90,11 +84,8 @@ def test_data_row_update(client, rand_gen):
     data_row.update(external_id=external_id_2)
     assert data_row.external_id == external_id_2
 
-    dataset.delete()
 
-
-def test_data_row_filtering_sorting(client, rand_gen):
-    dataset = client.create_dataset(name=rand_gen(str))
+def test_data_row_filtering_sorting(dataset, rand_gen):
     task = dataset.create_data_rows([
         {DataRow.row_data: IMG_URL, DataRow.external_id: "row1"},
         {DataRow.row_data: IMG_URL, DataRow.external_id: "row2"},
@@ -115,11 +106,8 @@ def test_data_row_filtering_sorting(client, rand_gen):
     assert list(dataset.data_rows(order_by=DataRow.external_id.asc)) == [row1, row2]
     assert list(dataset.data_rows(order_by=DataRow.external_id.desc)) == [row2, row1]
 
-    dataset.delete()
-
 
-def test_data_row_deletion(client, rand_gen):
-    dataset = client.create_dataset(name=rand_gen(str))
+def test_data_row_deletion(dataset, rand_gen):
     task = dataset.create_data_rows([
         {DataRow.row_data: IMG_URL, DataRow.external_id: str(i)}
         for i in range(10)])
@@ -142,5 +130,3 @@ def test_data_row_deletion(client, rand_gen):
 
     data_rows = list(dataset.data_rows())
     assert {dr.external_id for dr in data_rows} == expected
-
-    dataset.delete()

tests/integration/test_dataset.py

@@ -62,8 +62,7 @@ def test_dataset_filtering(client, rand_gen):
     d2.delete()
 
 
-def test_get_data_row_for_external_id(client, rand_gen):
-    dataset = client.create_dataset(name=rand_gen(str))
+def test_get_data_row_for_external_id(dataset, rand_gen):
     external_id = rand_gen(str)
 
     with pytest.raises(ResourceNotFoundError):

tests/integration/test_dates.py

@@ -1,19 +1,15 @@
 from datetime import datetime, timedelta, timezone
 
 
-def test_dates(client):
-    project = client.create_project(name="Test")
+def test_dates(project):
     assert isinstance(project.created_at, datetime)
     assert isinstance(project.updated_at, datetime)
 
     project.update(setup_complete=datetime.now())
     assert isinstance(project.setup_complete, datetime)
 
-    project.delete()
 
-
-def test_utc_conversion(client):
-    project = client.create_project(name="Test")
+def test_utc_conversion(project):
     assert isinstance(project.created_at, datetime)
     assert project.created_at.tzinfo == timezone.utc
 
@@ -29,5 +25,3 @@ def test_utc_conversion(client):
     project.update(setup_complete=datetime.utcnow().replace(tzinfo=tz))
     diff = datetime.utcnow() - project.setup_complete.replace(tzinfo=None)
     assert diff > timedelta(hours=5, minutes=58)
-
-    project.delete()

tests/integration/test_label.py

@@ -4,20 +4,13 @@
 import requests
 
 from labelbox import Label
-from labelbox.exceptions import InvalidQueryError
 
 
 IMG_URL = "https://picsum.photos/200/300"
 
 
-def test_labels(client, rand_gen):
-    project = client.create_project(name=rand_gen(str))
-    dataset = client.create_dataset(name=rand_gen(str), projects=project)
-
-    data_row = dataset.create_data_row(row_data=IMG_URL)
-
-    label = project.create_label(data_row=data_row, label="test",
-                                 seconds_to_label=0.0)
+def test_labels(label_pack):
+    project, dataset, data_row, label = label_pack
 
     # Labels are not visible in the project immediately.
     time.sleep(10)
@@ -36,15 +29,9 @@ def test_labels(client, rand_gen):
     assert list(project.labels()) == []
     assert list(data_row.labels()) == []
 
-    dataset.delete()
-    project.delete()
-
 
-def test_label_export(client, rand_gen):
-    project = client.create_project(name=rand_gen(str))
-    dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
-    project.create_label(data_row=data_row, label="l1", seconds_to_label=0.2)
+def test_label_export(label_pack):
+    project, dataset, data_row, label = label_pack
     project.create_label(data_row=data_row, label="l2", seconds_to_label=0.3)
 
     exported_labels_url = project.export_labels(5)
@@ -54,29 +41,13 @@ def test_label_export(client, rand_gen):
         # TODO check content
         assert False
 
-    dataset.delete()
-    project.delete()
-
 
-def test_label_update(client, rand_gen):
-    project = client.create_project(name=rand_gen(str))
-    dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
-
-    label = project.create_label(data_row=data_row, label="l1", seconds_to_label=0.0)
+def test_label_update(label_pack):
+    project, dataset, data_row, label = label_pack
 
-    assert label.label == "l1"
     label.update(label="something else")
     assert label.label == "something else"
 
-    # Check the label got updated server-side
-    # Labels are not visible in the project immediately.
-    time.sleep(10)
-    assert list(project.labels())[0].label == "something else"
-
-    dataset.delete()
-    project.delete()
-
 
 def test_label_filter_order(client, rand_gen):
     project = client.create_project(name=rand_gen(str))
@@ -110,9 +81,8 @@ def test_label_filter_order(client, rand_gen):
     project.delete()
 
 
-def test_label_bulk_deletion(client, rand_gen):
-    project = client.create_project(name=rand_gen(str))
-    dataset = client.create_dataset(name=rand_gen(str), projects=project)
+def test_label_bulk_deletion(project, rand_gen):
+    dataset = project.client.create_dataset(name=rand_gen(str), projects=project)
     row_1 = dataset.create_data_row(row_data=IMG_URL)
     row_2 = dataset.create_data_row(row_data=IMG_URL)
 
@@ -130,4 +100,3 @@ def test_label_bulk_deletion(client, rand_gen):
     assert set(project.labels()) == {l2}
 
     dataset.delete()
-    project.delete()

tests/integration/test_labeler_performance.py

@@ -2,24 +2,15 @@
 import time
 
 
-IMG_URL = "https://picsum.photos/200/300"
-
-
-def test_labeler_performance(client, rand_gen):
-    project = client.create_project(name=rand_gen(str))
-    assert list(project.labeler_performance()) == []
-
-    dataset = client.create_dataset(name=rand_gen(str), projects=project)
-    data_row = dataset.create_data_row(row_data=IMG_URL)
-    label = project.create_label(data_row=data_row, label="test",
-                                 seconds_to_label=0.0)
+def test_labeler_performance(label_pack):
+    project, dataset, data_row, label = label_pack
     # Sleep a bit as it seems labeler performance isn't updated immediately.
     time.sleep(10)
 
     labeler_performance = list(project.labeler_performance())
     assert len(labeler_performance) == 1
     my_performance = labeler_performance[0]
-    assert my_performance.user == client.get_user()
+    assert my_performance.user == project.client.get_user()
     assert my_performance.count == 1
     assert isinstance(my_performance.last_activity_time, datetime)
     now_utc = datetime.now().astimezone(timezone.utc)

tests/integration/test_labeling_frontend.py

@@ -1,5 +1,3 @@
-import pytest
-
 from labelbox import LabelingFrontend
 
 
@@ -13,11 +11,10 @@ def test_get_labeling_frontends(client):
     assert len(single) == 1
 
 
-def test_labeling_frontend_connecting_to_project(client, rand_gen):
-    project = client.create_project(name=rand_gen(str))
+def test_labeling_frontend_connecting_to_project(project):
     assert project.labeling_frontend() == None
 
-    frontend = list(client.get_labeling_frontends())[0]
+    frontend = list(project.client.get_labeling_frontends())[0]
 
     project.labeling_frontend.connect(frontend)
     assert project.labeling_frontend() == frontend
@@ -26,5 +23,3 @@ def test_labeling_frontend_connecting_to_project(client, rand_gen):
     project.labeling_frontend.disconnect(frontend)
     assert project.labeling_frontend() == None
     assert project not in set(frontend.projects())
-
-    project.delete()