Replacing configured_project

Author: Val Brodsky

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-addopts = -s -vv
+addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')

tests/conftest.py

@@ -12,7 +12,7 @@
 ]
 
 
-@pytest.fixture
+@pytest.fixture(scope="session")
 def rand_gen():
 
     def gen(field_type):

tests/integration/annotation_import/conftest.py

@@ -211,7 +211,7 @@ def annotations_by_data_type_v2(
     }
 
 
-@pytest.fixture
+@pytest.fixture(scope='session')
 def ontology():
     bbox_tool_with_nested_text = {
         'required':
@@ -479,48 +479,45 @@ def func(project):
 
 
 @pytest.fixture
-def initial_dataset(client, rand_gen):
-    dataset = client.create_dataset(name=rand_gen(str))
-    yield dataset
-    dataset.delete()
-
-
-@pytest.fixture
-def hardcoded_datarow_id():
-    data_row_id = 'ck8q9q9qj00003g5z3q1q9q9q'
+def configured_project_datarow_id(configured_project):
 
     def get_data_row_id(indx=0):
-        return data_row_id
+        return configured_project.data_row_ids[indx]
 
     yield get_data_row_id
 
 
 @pytest.fixture
-def configured_project_datarow_id(configured_project):
+def configured_project_one_datarow_id(configured_project_with_one_data_row):
 
     def get_data_row_id(indx=0):
-        return configured_project.data_row_ids[indx]
+        return configured_project_with_one_data_row.data_row_ids[0]
 
     yield get_data_row_id
 
 
 @pytest.fixture
-def configured_project(configured_project_without_data_rows, initial_dataset,
-                       ontology, rand_gen, image_url):
+def configured_project(client, initial_dataset, ontology, rand_gen, image_url):
     start_time = time.time()
     dataset = initial_dataset
-    project = configured_project_without_data_rows
+    project = client.create_project(name=rand_gen(str),
+                                    queue_mode=QueueMode.Batch)
+    editor = list(
+        client.get_labeling_frontends(
+            where=LabelingFrontend.name == "editor"))[0]
+    project.setup(editor, ontology)
+    num_rows = 0
 
     data_row_ids = []
-    # print("Before creating data rows ", time.time() - start_time)
-    num_rows = 0
+
     for _ in range(len(ontology['tools']) + len(ontology['classifications'])):
         data_row_ids.append(dataset.create_data_row(row_data=image_url).uid)
         num_rows += 1
-    # print("After creating data rows ", time.time() - start_time)
-
-    pytest.data_row_report['times'] += time.time() - start_time
-    pytest.data_row_report['num_rows'] += num_rows
+    project._wait_until_data_rows_are_processed(data_row_ids=data_row_ids,
+                                                sleep_interval=3)
+    if pytest.data_row_report:
+        pytest.data_row_report['times'] += time.time() - start_time
+        pytest.data_row_report['num_rows'] += num_rows
     project.create_batch(
         rand_gen(str),
         data_row_ids,  # sample of data row objects
@@ -580,15 +577,36 @@ def dataset_conversation_entity(client, rand_gen, conversation_entity_data_row,
 
 
 @pytest.fixture
-def configured_project_without_data_rows(client, ontology, rand_gen):
+def configured_project_with_one_data_row(client, ontology, rand_gen,
+                                         initial_dataset, image_url):
+    start_time = time.time()
+
     project = client.create_project(name=rand_gen(str),
                                     description=rand_gen(str),
                                     queue_mode=QueueMode.Batch)
     editor = list(
         client.get_labeling_frontends(
             where=LabelingFrontend.name == "editor"))[0]
     project.setup(editor, ontology)
+
+    data_row = initial_dataset.create_data_row(row_data=image_url)
+    data_row_ids = [data_row.uid]
+    project._wait_until_data_rows_are_processed(data_row_ids=data_row_ids,
+                                                sleep_interval=3)
+
+    if pytest.data_row_report:
+        pytest.data_row_report['times'] += time.time() - start_time
+        pytest.data_row_report['num_rows'] += 1
+    batch = project.create_batch(
+        rand_gen(str),
+        data_row_ids,  # sample of data row objects
+        5  # priority between 1(Highest) - 5(lowest)
+    )
+    project.data_row_ids = data_row_ids
+
     yield project
+
+    batch.delete()
     project.delete()
 
 
@@ -597,16 +615,20 @@ def configured_project_without_data_rows(client, ontology, rand_gen):
 # In an example of a 'rectangle' we have extended to support multiple instances of the same tool type
 # TODO: we will support this approach in the future for all tools
 @pytest.fixture
-def prediction_id_mapping(configured_project_without_data_rows, ontology,
-                          request):
+def prediction_id_mapping(ontology, request):
     # Maps tool types to feature schema ids
     if 'configured_project' in request.fixturenames:
         data_row_id_factory = request.getfixturevalue(
             'configured_project_datarow_id')
-        project = configured_project
-    else:
+        project = request.getfixturevalue('configured_project')
+    elif 'hardcoded_datarow_id' in request.fixturenames:
         data_row_id_factory = request.getfixturevalue('hardcoded_datarow_id')
-        project = configured_project_without_data_rows
+        project = request.getfixturevalue('configured_project_with_ontology')
+    else:
+        data_row_id_factory = request.getfixturevalue(
+            'configured_project_one_datarow_id')
+        project = request.getfixturevalue(
+            'configured_project_with_one_data_row')
 
     ontology = project.ontology().normalized
 
@@ -646,46 +668,6 @@ def prediction_id_mapping(configured_project_without_data_rows, ontology,
     return result
 
 
-@pytest.fixture
-def prediction_id_mapping_datarow_id():
-    # Maps tool types to feature schema ids
-    data_row_id = 'ck8q9q9qj00003g5z3q1q9q9q'
-    result = {}
-
-    for _, tool in enumerate(ontology['tools'] + ontology['classifications']):
-        if 'tool' in tool:
-            tool_type = tool['tool']
-        else:
-            tool_type = tool[
-                'type'] if 'scope' not in tool else f"{tool['type']}_{tool['scope']}"  # so 'checklist' of 'checklist_index'
-
-        # TODO: remove this once we have a better way to associate multiple tools instances with a single tool type
-        if tool_type == 'rectangle':
-            value = {
-                "uuid": str(uuid.uuid4()),
-                "schemaId": tool['featureSchemaId'],
-                "name": tool['name'],
-                "dataRow": {
-                    "id": data_row_id,
-                },
-                'tool': tool
-            }
-            if tool_type not in result:
-                result[tool_type] = []
-            result[tool_type].append(value)
-        else:
-            result[tool_type] = {
-                "uuid": str(uuid.uuid4()),
-                "schemaId": tool['featureSchemaId'],
-                "name": tool['name'],
-                "dataRow": {
-                    "id": data_row_id,
-                },
-                'tool': tool
-            }
-    return result
-
-
 @pytest.fixture
 def polygon_inference(prediction_id_mapping):
     polygon = prediction_id_mapping['polygon'].copy()
@@ -1079,7 +1061,6 @@ def model_run_with_training_metadata(rand_gen, model):
 @pytest.fixture
 def model_run_with_data_rows(client, configured_project, model_run_predictions,
                              model_run, wait_for_label_processing):
-    start_time = time.time()
     configured_project.enable_model_assisted_labeling()
 
     upload_task = LabelImport.create_from_objects(
@@ -1093,7 +1074,6 @@ def model_run_with_data_rows(client, configured_project, model_run_predictions,
     labels = wait_for_label_processing(configured_project)
     label_ids = [label.uid for label in labels]
     model_run.upsert_labels(label_ids)
-    print(f"model_run_with_data_rows: {time.time() - start_time}")
     yield model_run
     model_run.delete()
     # TODO: Delete resources when that is possible ..

tests/integration/annotation_import/test_bulk_import_request.py

@@ -51,14 +51,14 @@ def test_validate_file(project_with_ontology):
         #Schema ids shouldn't match
 
 
-def test_create_from_objects(configured_project_without_data_rows, predictions,
+def test_create_from_objects(configured_project_with_one_data_row, predictions,
                              annotation_import_test_helpers):
     name = str(uuid.uuid4())
 
-    bulk_import_request = configured_project_without_data_rows.upload_annotations(
+    bulk_import_request = configured_project_with_one_data_row.upload_annotations(
         name=name, annotations=predictions)
 
-    assert bulk_import_request.project() == configured_project_without_data_rows
+    assert bulk_import_request.project() == configured_project_with_one_data_row
     assert bulk_import_request.name == name
     assert bulk_import_request.error_file_url is None
     assert bulk_import_request.status_file_url is None
@@ -105,34 +105,33 @@ def test_create_from_local_file(tmp_path, predictions, configured_project,
         bulk_import_request.input_file_url, predictions)
 
 
-def test_get(client, configured_project_without_data_rows):
+def test_get(client, configured_project_with_one_data_row):
     name = str(uuid.uuid4())
     url = "https://storage.googleapis.com/labelbox-public-bucket/predictions_test_v2.ndjson"
-    configured_project_without_data_rows.upload_annotations(name=name,
+    configured_project_with_one_data_row.upload_annotations(name=name,
                                                             annotations=url,
                                                             validate=False)
 
     bulk_import_request = BulkImportRequest.from_name(
-        client, project_id=configured_project_without_data_rows.uid, name=name)
+        client, project_id=configured_project_with_one_data_row.uid, name=name)
 
-    assert bulk_import_request.project() == configured_project_without_data_rows
+    assert bulk_import_request.project() == configured_project_with_one_data_row
     assert bulk_import_request.name == name
     assert bulk_import_request.input_file_url == url
     assert bulk_import_request.error_file_url is None
     assert bulk_import_request.status_file_url is None
     assert bulk_import_request.state == BulkImportRequestState.RUNNING
 
 
-def test_validate_ndjson(tmp_path, configured_project):
+def test_validate_ndjson(tmp_path, configured_project_with_one_data_row):
     file_name = f"broken.ndjson"
     file_path = tmp_path / file_name
     with file_path.open("w") as f:
         f.write("test")
 
     with pytest.raises(ValueError):
-        configured_project.upload_annotations(name="name",
-                                              validate=True,
-                                              annotations=str(file_path))
+        configured_project_with_one_data_row.upload_annotations(
+            name="name", validate=True, annotations=str(file_path))
 
 
 def test_validate_ndjson_uuid(tmp_path, configured_project, predictions):
@@ -159,11 +158,11 @@ def test_validate_ndjson_uuid(tmp_path, configured_project, predictions):
 
 @pytest.mark.slow
 def test_wait_till_done(rectangle_inference,
-                        configured_project_without_data_rows):
+                        configured_project_with_one_data_row):
     name = str(uuid.uuid4())
-    url = configured_project_without_data_rows.client.upload_data(
+    url = configured_project_with_one_data_row.client.upload_data(
         content=parser.dumps([rectangle_inference]), sign=True)
-    bulk_import_request = configured_project_without_data_rows.upload_annotations(
+    bulk_import_request = configured_project_with_one_data_row.upload_annotations(
         name=name, annotations=url, validate=False)
 
     assert len(bulk_import_request.inputs) == 1
@@ -298,7 +297,7 @@ def test_pdf_mal_bbox(client, configured_project_pdf):
     assert import_annotations.errors == []
 
 
-def test_pdf_document_entity(client, configured_project_without_data_rows,
+def test_pdf_document_entity(client, configured_project_with_one_data_row,
                              dataset_pdf_entity, rand_gen):
     # for content "Metal-insulator (MI) transitions have been one of the" in OCR JSON extract tests/assets/arxiv-pdf_data_99-word-token-pdfs_0801.3483-lb-textlayer.json
     document_text_selection = DocumentTextSelection(
@@ -322,7 +321,7 @@ def test_pdf_document_entity(client, configured_project_without_data_rows,
 
     labels = []
     _, data_row_uids = dataset_pdf_entity
-    configured_project_without_data_rows.create_batch(
+    configured_project_with_one_data_row.create_batch(
         rand_gen(str),
         data_row_uids,  # sample of data row objects
         5  # priority between 1(Highest) - 5(lowest)
@@ -337,7 +336,7 @@ def test_pdf_document_entity(client, configured_project_without_data_rows,
 
     import_annotations = MALPredictionImport.create_from_objects(
         client=client,
-        project_id=configured_project_without_data_rows.uid,
+        project_id=configured_project_with_one_data_row.uid,
         name=f"import {str(uuid.uuid4())}",
         predictions=labels)
     import_annotations.wait_until_done()
@@ -346,14 +345,14 @@ def test_pdf_document_entity(client, configured_project_without_data_rows,
 
 
 def test_nested_video_object_annotations(client,
-                                         configured_project_without_data_rows,
+                                         configured_project_with_one_data_row,
                                          video_data,
                                          bbox_video_annotation_objects,
                                          rand_gen):
     labels = []
     _, data_row_uids = video_data
-    configured_project_without_data_rows.update(media_type=MediaType.Video)
-    configured_project_without_data_rows.create_batch(
+    configured_project_with_one_data_row.update(media_type=MediaType.Video)
+    configured_project_with_one_data_row.create_batch(
         rand_gen(str),
         data_row_uids,  # sample of data row objects
         5  # priority between 1(Highest) - 5(lowest)
@@ -365,7 +364,7 @@ def test_nested_video_object_annotations(client,
                   annotations=bbox_video_annotation_objects))
     import_annotations = MALPredictionImport.create_from_objects(
         client=client,
-        project_id=configured_project_without_data_rows.uid,
+        project_id=configured_project_with_one_data_row.uid,
         name=f"import {str(uuid.uuid4())}",
         predictions=labels)
     import_annotations.wait_until_done()

tests/integration/annotation_import/test_conversation_import.py

@@ -7,7 +7,7 @@
 from labelbox.schema.annotation_import import MALPredictionImport
 
 
-def test_conversation_entity(client, configured_project_without_data_rows,
+def test_conversation_entity(client, configured_project_with_one_data_row,
                              dataset_conversation_entity, rand_gen):
 
     conversation_entity_annotation = ConversationEntity(start=0,
@@ -20,7 +20,7 @@ def test_conversation_entity(client, configured_project_without_data_rows,
     labels = []
     _, data_row_uids = dataset_conversation_entity
 
-    configured_project_without_data_rows.create_batch(
+    configured_project_with_one_data_row.create_batch(
         rand_gen(str),
         data_row_uids,  # sample of data row objects
         5  # priority between 1(Highest) - 5(lowest)
@@ -35,7 +35,7 @@ def test_conversation_entity(client, configured_project_without_data_rows,
 
     import_annotations = MALPredictionImport.create_from_objects(
         client=client,
-        project_id=configured_project_without_data_rows.uid,
+        project_id=configured_project_with_one_data_row.uid,
         name=f"import {str(uuid.uuid4())}",
         predictions=labels)