Convert tests that do now require many data rows prebuilt to a simpler project fixture

Author: Val Brodsky

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20
+addopts = -s -vv
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')

tests/integration/annotation_import/conftest.py

@@ -486,17 +486,12 @@ def initial_dataset(client, rand_gen):
 
 
 @pytest.fixture
-def configured_project(client, initial_dataset, ontology, rand_gen, image_url):
+def configured_project(client, configured_project_without_data_rows,
+                       initial_dataset, ontology, rand_gen, image_url):
     start_time = time.time()
     dataset = initial_dataset
-    project = client.create_project(
-        name=rand_gen(str),
-        queue_mode=QueueMode.Batch,
-    )
-    editor = list(
-        client.get_labeling_frontends(
-            where=LabelingFrontend.name == "editor"))[0]
-    project.setup(editor, ontology)
+    project = configured_project_without_data_rows
+
     data_row_ids = []
     # print("Before creating data rows ", time.time() - start_time)
     num_rows = 0

tests/integration/annotation_import/test_bulk_import_request.py

@@ -25,40 +25,40 @@
 """
 
 
-def test_create_from_url(configured_project):
+def test_create_from_url(project):
     name = str(uuid.uuid4())
     url = "https://storage.googleapis.com/labelbox-public-bucket/predictions_test_v2.ndjson"
 
-    bulk_import_request = configured_project.upload_annotations(name=name,
-                                                                annotations=url,
-                                                                validate=False)
+    bulk_import_request = project.upload_annotations(name=name,
+                                                     annotations=url,
+                                                     validate=False)
 
-    assert bulk_import_request.project() == configured_project
+    assert bulk_import_request.project() == project
     assert bulk_import_request.name == name
     assert bulk_import_request.input_file_url == url
     assert bulk_import_request.error_file_url is None
     assert bulk_import_request.status_file_url is None
     assert bulk_import_request.state == BulkImportRequestState.RUNNING
 
 
-def test_validate_file(configured_project):
+def test_validate_file(project_with_ontology):
     name = str(uuid.uuid4())
     url = "https://storage.googleapis.com/labelbox-public-bucket/predictions_test_v2.ndjson"
     with pytest.raises(MALValidationError):
-        configured_project.upload_annotations(name=name,
-                                              annotations=url,
-                                              validate=True)
+        project_with_ontology.upload_annotations(name=name,
+                                                 annotations=url,
+                                                 validate=True)
         #Schema ids shouldn't match
 
 
-def test_create_from_objects(configured_project, predictions,
+def test_create_from_objects(configured_project_without_data_rows, predictions,
                              annotation_import_test_helpers):
     name = str(uuid.uuid4())
 
-    bulk_import_request = configured_project.upload_annotations(
+    bulk_import_request = configured_project_without_data_rows.upload_annotations(
         name=name, annotations=predictions)
 
-    assert bulk_import_request.project() == configured_project
+    assert bulk_import_request.project() == configured_project_without_data_rows
     assert bulk_import_request.name == name
     assert bulk_import_request.error_file_url is None
     assert bulk_import_request.status_file_url is None
@@ -105,17 +105,17 @@ def test_create_from_local_file(tmp_path, predictions, configured_project,
         bulk_import_request.input_file_url, predictions)
 
 
-def test_get(client, configured_project):
+def test_get(client, configured_project_without_data_rows):
     name = str(uuid.uuid4())
     url = "https://storage.googleapis.com/labelbox-public-bucket/predictions_test_v2.ndjson"
-    configured_project.upload_annotations(name=name,
-                                          annotations=url,
-                                          validate=False)
+    configured_project_without_data_rows.upload_annotations(name=name,
+                                                            annotations=url,
+                                                            validate=False)
 
     bulk_import_request = BulkImportRequest.from_name(
-        client, project_id=configured_project.uid, name=name)
+        client, project_id=configured_project_without_data_rows.uid, name=name)
 
-    assert bulk_import_request.project() == configured_project
+    assert bulk_import_request.project() == configured_project_without_data_rows
     assert bulk_import_request.name == name
     assert bulk_import_request.input_file_url == url
     assert bulk_import_request.error_file_url is None
@@ -158,14 +158,13 @@ def test_validate_ndjson_uuid(tmp_path, configured_project, predictions):
 
 
 @pytest.mark.slow
-def test_wait_till_done(rectangle_inference, configured_project):
+def test_wait_till_done(rectangle_inference,
+                        configured_project_without_data_rows):
     name = str(uuid.uuid4())
-    url = configured_project.client.upload_data(content=parser.dumps(
-        [rectangle_inference]),
-                                                sign=True)
-    bulk_import_request = configured_project.upload_annotations(name=name,
-                                                                annotations=url,
-                                                                validate=False)
+    url = configured_project_without_data_rows.client.upload_data(
+        content=parser.dumps([rectangle_inference]), sign=True)
+    bulk_import_request = configured_project_without_data_rows.upload_annotations(
+        name=name, annotations=url, validate=False)
 
     assert len(bulk_import_request.inputs) == 1
     bulk_import_request.wait_until_done()

tests/integration/annotation_import/test_data_types.py

@@ -125,7 +125,6 @@ def create_data_row_for_project(project, dataset, data_row_ndjson, batch_name):
         [data_row.uid],  # sample of data row objects
         5  # priority between 1(Highest) - 5(lowest)
     )
-    project.data_row_ids.append(data_row.uid)
 
     return data_row
 
@@ -135,12 +134,12 @@ def create_data_row_for_project(project, dataset, data_row_ndjson, batch_name):
     AudioData, ConversationData, DicomData, DocumentData, HTMLData, ImageData,
     TextData
 ])
-def test_import_data_types(client, configured_project, initial_dataset,
-                           rand_gen, data_row_json_by_data_type,
-                           annotations_by_data_type, data_type_class):
+def test_import_data_types(client, project, initial_dataset, rand_gen,
+                           data_row_json_by_data_type, annotations_by_data_type,
+                           data_type_class):
 
-    project = configured_project
-    project_id = configured_project.uid
+    project = project
+    project_id = project.uid
     dataset = initial_dataset
 
     set_project_media_type_from_data_type(project, data_type_class)
@@ -261,11 +260,11 @@ def test_import_data_types_v2(client, configured_project, initial_dataset,
 
 
 @pytest.mark.parametrize('data_type, data_class, annotations', test_params)
-def test_import_label_annotations(client, configured_project, initial_dataset,
-                                  data_row_json_by_data_type, data_type,
-                                  data_class, annotations, rand_gen):
+def test_import_label_annotations(client, configured_project_without_data_rows,
+                                  initial_dataset, data_row_json_by_data_type,
+                                  data_type, data_class, annotations, rand_gen):
 
-    project = configured_project
+    project = configured_project_without_data_rows
     dataset = initial_dataset
     set_project_media_type_from_data_type(project, data_class)
 
@@ -297,7 +296,8 @@ def test_import_label_annotations(client, configured_project, initial_dataset,
     assert export_task.errors is None
     expected_annotations = get_annotation_comparison_dicts_from_labels(labels)
     actual_annotations = get_annotation_comparison_dicts_from_export(
-        export_task.result, data_row.uid, configured_project.uid)
+        export_task.result, data_row.uid,
+        configured_project_without_data_rows.uid)
     assert actual_annotations == expected_annotations
     data_row.delete()
 

tests/integration/conftest.py

@@ -3,6 +3,7 @@
 import json
 import os
 import re
+import sys
 import time
 import uuid
 from enum import Enum
@@ -390,9 +391,21 @@ def initial_dataset(client, rand_gen):
 
 
 @pytest.fixture
-def configured_project(project, initial_dataset, client, rand_gen, image_url):
+def project_with_ontology(project):
+    editor = list(
+        project.client.get_labeling_frontends(
+            where=LabelingFrontend.name == "editor"))[0]
+    empty_ontology = {"tools": [], "classifications": []}
+    project.setup(editor, empty_ontology)
+    yield project
+
+
+@pytest.fixture
+def configured_project(project_with_ontology, initial_dataset, rand_gen,
+                       image_url):
     dataset = initial_dataset
     data_row_id = dataset.create_data_row(row_data=image_url).uid
+    project = project_with_ontology
 
     project.create_batch(
         rand_gen(str),
@@ -401,14 +414,7 @@ def configured_project(project, initial_dataset, client, rand_gen, image_url):
     )
     project.data_row_ids = [data_row_id]
 
-    editor = list(
-        project.client.get_labeling_frontends(
-            where=LabelingFrontend.name == "editor"))[0]
-    empty_ontology = {"tools": [], "classifications": []}
-    project.setup(editor, empty_ontology)
     yield project
-    dataset.delete()
-    project.delete()
 
 
 @pytest.fixture
@@ -833,6 +839,8 @@ def pytest_fixture_setup(fixturedef, request):
 
 @pytest.fixture(scope='session', autouse=True)
 def print_perf_summary():
+    print("Starting measurements\n", file=sys.stderr)
+
     yield
 
     sorted_dict = dict(
@@ -841,5 +849,6 @@ def print_perf_summary():
     slowest_fixtures = [
         (aaa, sorted_dict[aaa]) for aaa in islice(sorted_dict, num_of_entries)
     ]
-    print("\nTop slowest fixtures:\n", slowest_fixtures)
-    print("Data row report:\n", pytest.data_row_report)
+    print("\nTop slowest fixtures:\n", slowest_fixtures, file=sys.stderr)
+    print("Data row report:\n", pytest.data_row_report, file=sys.stderr)
+    # assert False

tests/integration/test_project.py

@@ -171,15 +171,15 @@ def test_attach_instructions(client, project):
 
 @pytest.mark.skipif(condition=os.environ['LABELBOX_TEST_ENVIRON'] == "onprem",
                     reason="new mutation does not work for onprem")
-def test_html_instructions(configured_project):
+def test_html_instructions(project_with_ontology):
     html_file_path = '/tmp/instructions.html'
     sample_html_str = "<html></html>"
 
     with open(html_file_path, 'w') as file:
         file.write(sample_html_str)
 
-    configured_project.upsert_instructions(html_file_path)
-    updated_ontology = configured_project.ontology().normalized
+    project_with_ontology.upsert_instructions(html_file_path)
+    updated_ontology = project_with_ontology.ontology().normalized
 
     instructions = updated_ontology.pop('projectInstructions')
     assert requests.get(instructions).text == sample_html_str
@@ -200,10 +200,6 @@ def test_same_ontology_after_instructions(
     assert instructions is not None
 
 
-def test_queue_mode(configured_project: Project):
-    assert configured_project.queue_mode == QueueMode.Batch
-
-
 def test_batches(project: Project, dataset: Dataset, image_url):
     task = dataset.create_data_rows([
         {
@@ -243,9 +239,9 @@ def test_create_batch_with_global_keys_async(project: Project, data_rows):
     assert batch_data_rows == set(data_rows)
 
 
-def test_media_type(client, configured_project: Project, rand_gen):
+def test_media_type(client, project: Project, rand_gen):
     # Existing project with no media_type
-    assert isinstance(configured_project.media_type, MediaType)
+    assert isinstance(project.media_type, MediaType)
 
     # Update test
     project = client.create_project(name=rand_gen(str))