[AL-6133] Optimize and stabilize sdk integration tests part I (#1195)

Author: vbrodsky

tests/integration/test_client_errors.py

@@ -77,7 +77,17 @@ def test_network_error(client):
         client.create_project(name="Project name")
 
 
-def test_invalid_attribute_error(client, rand_gen):
+@pytest.fixture
+def project_for_test_invalid_attribute_error(client):
+    project = client.create_project(name="Project name")
+    yield project
+    project.delete()
+
+
+def test_invalid_attribute_error(client, rand_gen,
+                                 project_for_test_invalid_attribute_error):
+    project = project_for_test_invalid_attribute_error
+
     # Creation
     with pytest.raises(labelbox.exceptions.InvalidAttributeError) as excinfo:
         client.create_project(name="Name", invalid_field="Whatever")
@@ -109,8 +119,6 @@ def test_invalid_attribute_error(client, rand_gen):
     assert excinfo.value.db_object_type == Project
     assert excinfo.value.field == {User.email}
 
-    project.delete()
-
 
 @pytest.mark.skip("timeouts cause failure before rate limit")
 def test_api_limit_error(client):

tests/integration/test_data_rows.py

@@ -201,7 +201,8 @@ def test_data_row_bulk_creation(dataset, rand_gen, image_url):
     url = ({data_row.row_data for data_row in data_rows} - {image_url}).pop()
     assert requests.get(url).content == data
 
-    data_rows[0].delete()
+    for dr in data_rows:
+        dr.delete()
 
 
 @pytest.mark.slow
@@ -589,13 +590,23 @@ def test_data_row_filtering_sorting(dataset, image_url):
     assert row2.external_id == "row2"
 
 
-def test_data_row_deletion(dataset, image_url):
+@pytest.fixture
+def create_datarows_for_data_row_deletion(dataset, image_url):
     task = dataset.create_data_rows([{
         DataRow.row_data: image_url,
         DataRow.external_id: str(i)
     } for i in range(10)])
     task.wait_till_done()
 
+    data_rows = list(dataset.data_rows())
+
+    yield data_rows
+    for dr in data_rows:
+        dr.delete()
+
+
+def test_data_row_deletion(dataset, create_datarows_for_data_row_deletion):
+    create_datarows_for_data_row_deletion
     data_rows = list(dataset.data_rows())
     expected = set(map(str, range(10)))
     assert {dr.external_id for dr in data_rows} == expected

tests/integration/test_dataset.py

@@ -44,19 +44,26 @@ def test_dataset(client, rand_gen):
         dataset = client.get_dataset(dataset.uid)
 
 
-def test_dataset_filtering(client, rand_gen):
+@pytest.fixture
+def dataset_for_filtering(client, rand_gen):
     name_1 = rand_gen(str)
     name_2 = rand_gen(str)
     d1 = client.create_dataset(name=name_1)
     d2 = client.create_dataset(name=name_2)
 
-    assert list(client.get_datasets(where=Dataset.name == name_1)) == [d1]
-    assert list(client.get_datasets(where=Dataset.name == name_2)) == [d2]
+    yield name_1, d1, name_2, d2
 
     d1.delete()
     d2.delete()
 
 
+def test_dataset_filtering(client, dataset_for_filtering):
+    name_1, d1, name_2, d2 = dataset_for_filtering
+
+    assert list(client.get_datasets(where=Dataset.name == name_1)) == [d1]
+    assert list(client.get_datasets(where=Dataset.name == name_2)) == [d2]
+
+
 def test_get_data_row_for_external_id(dataset, rand_gen, image_url):
     external_id = rand_gen(str)
 

tests/integration/test_filtering.py

@@ -5,16 +5,29 @@
 from labelbox.schema.queue_mode import QueueMode
 
 
-# Avoid assertions using equality to prevent intermittent failures due to
-# other builds simultaneously adding projects to test org
-def test_where(client, image_url, rand_gen):
+@pytest.fixture
+def project_to_test_where(client, rand_gen):
     p_a_name = f"a-{rand_gen(str)}"
     p_b_name = f"b-{rand_gen(str)}"
     p_c_name = f"c-{rand_gen(str)}"
+
     p_a = client.create_project(name=p_a_name, queue_mode=QueueMode.Batch)
     p_b = client.create_project(name=p_b_name, queue_mode=QueueMode.Batch)
     p_c = client.create_project(name=p_c_name, queue_mode=QueueMode.Batch)
 
+    yield p_a, p_b, p_c
+
+    p_a.delete()
+    p_b.delete()
+    p_c.delete()
+
+
+# Avoid assertions using equality to prevent intermittent failures due to
+# other builds simultaneously adding projects to test org
+def test_where(client, image_url, project_to_test_where, rand_gen):
+    p_a, p_b, p_c = project_to_test_where
+    p_a_name, p_b_name, p_c_name = [p.name for p in [p_a, p_b, p_c]]
+
     def _get(f, where=None):
         date_where = Project.created_at >= p_a.created_at
         where = date_where if where is None else where & date_where
@@ -64,10 +77,6 @@ def get(where=None):
     assert {p_a.uid, p_b.uid}.issubset(le_b) and p_c.uid not in le_b
 
     batch.delete()
-    p_a.delete()
-    p_b.delete()
-    p_c.delete()
-    dataset.delete()
 
 
 def test_unsupported_where(client):

tests/integration/test_labeling_parameter_overrides.py

@@ -66,5 +66,3 @@ def test_labeling_parameter_overrides(consensus_project, initial_dataset,
         project.set_labeling_parameter_overrides(data)
     assert str(exc_info.value) == \
         f"Priority must be greater than 0 for data_row {data_rows[2]}. Index: 0"
-
-    dataset.delete()

tests/integration/test_ontology.py

@@ -5,6 +5,8 @@
 import json
 import time
 
+from labelbox.schema.queue_mode import QueueMode
+
 
 def test_feature_schema_is_not_archived(client, ontology):
     feature_schema_to_check = ontology.normalized['tools'][0]
@@ -85,6 +87,7 @@ def test_deletes_an_ontology(client):
 
 def test_cant_delete_an_ontology_with_project(client):
     project = client.create_project(name="test project",
+                                    queue_mode=QueueMode.Batch,
                                     media_type=MediaType.Image)
     tool = client.upsert_feature_schema(point.asdict())
     feature_schema_id = tool.normalized['featureSchemaId']
@@ -155,6 +158,7 @@ def test_does_not_include_used_ontologies(client):
         feature_schema_ids=[feature_schema_id],
         media_type=MediaType.Image)
     project = client.create_project(name="test project",
+                                    queue_mode=QueueMode.Batch,
                                     media_type=MediaType.Image)
     project.setup_editor(ontology_with_project)
     unused_ontologies = client.get_unused_ontologies()

tests/integration/test_pagination.py

@@ -1,10 +1,30 @@
 from copy import copy
+import time
 
+import pytest
 
-def test_get_one_and_many_dataset_order(client):
-    paginator = client.get_datasets()
+from labelbox.schema.dataset import Dataset
+
+
+@pytest.fixture
+def data_for_dataset_order_test(client, rand_gen):
+    name = rand_gen(str)
+    dataset1 = client.create_dataset(name=name)
+    dataset2 = client.create_dataset(name=name)
+
+    yield name
+
+    dataset1.delete()
+    dataset2.delete()
+
+
+def test_get_one_and_many_dataset_order(client, data_for_dataset_order_test):
+    name = data_for_dataset_order_test
+
+    paginator = client.get_datasets(where=Dataset.name == name)
     # confirm get_one returns first dataset
     all_datasets = list(paginator)
+    assert len(all_datasets) == 2
     get_one_dataset = copy(paginator).get_one()
     assert get_one_dataset.uid == all_datasets[0].uid
 

tests/integration/test_project.py

@@ -213,7 +213,25 @@ def test_project_export_v2_datarow_list(
                ]) == set(global_keys[:datarow_filter_size])
 
 
-def test_update_project_resource_tags(client, rand_gen):
+@pytest.fixture
+def data_for_project_test(client, rand_gen):
+    projects = []
+
+    def _create_project(name: str = None):
+        if name is None:
+            name = rand_gen(str)
+        project = client.create_project(name=name)
+        projects.append(project)
+        return project
+
+    yield _create_project
+
+    for project in projects:
+        project.delete()
+
+
+def test_update_project_resource_tags(client, rand_gen, data_for_project_test):
+    p1 = data_for_project_test()
 
     def delete_tag(tag_id: str):
         """Deletes a tag given the tag uid. Currently internal use only so this is not public"""
@@ -226,15 +244,9 @@ def delete_tag(tag_id: str):
         """, {"tag_id": tag_id})
         return res
 
-    before = list(client.get_projects())
-    for o in before:
-        assert isinstance(o, Project)
-
     org = client.get_organization()
     assert org.uid is not None
 
-    project_name = rand_gen(str)
-    p1 = client.create_project(name=project_name)
     assert p1.uid is not None
 
     colorA = "#ffffff"
@@ -273,18 +285,15 @@ def delete_tag(tag_id: str):
     delete_tag(tagB.uid)
 
 
-def test_project_filtering(client, rand_gen):
+def test_project_filtering(client, rand_gen, data_for_project_test):
     name_1 = rand_gen(str)
+    p1 = data_for_project_test(name_1)
     name_2 = rand_gen(str)
-    p1 = client.create_project(name=name_1)
-    p2 = client.create_project(name=name_2)
+    p2 = data_for_project_test(name_2)
 
     assert list(client.get_projects(where=Project.name == name_1)) == [p1]
     assert list(client.get_projects(where=Project.name == name_2)) == [p2]
 
-    p1.delete()
-    p2.delete()
-
 
 def test_upsert_review_queue(project):
     project.upsert_review_queue(0.6)