Merge pull request #831 from Labelbox/imuhammad/AL-3976-parallelizable-dataset-tests

[AL-4418] [AL-3976] Make dataset tests parallelizable

Author: whistler

Makefile

@@ -21,7 +21,7 @@ test-staging: build
 		-e LABELBOX_TEST_ENVIRON="staging" \
 		-e DA_GCP_LABELBOX_API_KEY=${DA_GCP_LABELBOX_API_KEY} \
 		-e LABELBOX_TEST_API_KEY_STAGING=${LABELBOX_TEST_API_KEY_STAGING} \
-		local/labelbox-python:test pytest $(PATH_TO_TEST)
+		local/labelbox-python:test pytest -n 10 $(PATH_TO_TEST)
 
 test-prod: build
 	docker run -it -v ${PWD}:/usr/src -w /usr/src \

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-addopts = -s -vv -x --reruns 5 --reruns-delay 10 --durations=20
+addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')

requirements.txt

@@ -13,4 +13,5 @@ typeguard
 imagesize
 pyproj
 pygeotile
-typing-extensions
+typing-extensions
+pytest-xdist

tests/integration/annotation_import/test_model.py

@@ -1,31 +1,26 @@
+import pytest
+
 from labelbox import Model
+from labelbox.exceptions import ResourceNotFoundError
 
 
 def test_model(client, configured_project, rand_gen):
-    before = list(client.get_models())
-    for m in before:
+    # Get all
+    models = list(client.get_models())
+    for m in models:
         assert isinstance(m, Model)
 
+    # Create
     ontology = configured_project.ontology()
-
     data = {"name": rand_gen(str), "ontology_id": ontology.uid}
     model = client.create_model(data["name"], data["ontology_id"])
     assert model.name == data["name"]
 
-    after = list(client.get_models())
-    assert len(after) == len(before) + 1
-    assert model in after
-
+    # Get one
     model = client.get_model(model.uid)
     assert model.name == data["name"]
 
-
-def test_model_delete(client, model):
-    before = list(client.get_models())
-
-    model = before[0]
+    # Delete
     model.delete()
-
-    after = list(client.get_models())
-
-    assert len(before) == len(after) + 1
+    with pytest.raises(ResourceNotFoundError):
+        client.get_model(model.uid)

tests/integration/annotation_import/test_model_run.py

@@ -52,8 +52,9 @@ def test_model_run_delete(client, model_run):
     models_after = list(client.get_models())
     model_after = models_after[0]
     after = list(model_after.model_runs())
+    after_uids = {mr.uid for mr in after}
 
-    assert len(before) == len(after) + 1
+    assert model_run.uid not in after_uids
 
 
 def test_model_run_update_config(model_run_with_training_metadata):
@@ -74,11 +75,8 @@ def test_model_run_get_config(model_run_with_training_metadata):
     assert res["batch_size"] == new_config["batch_size"]
 
 
-def test_model_run_data_rows_delete(client, model_run_with_model_run_data_rows):
-    models = list(client.get_models())
-    model = models[0]
-    model_runs = list(model.model_runs())
-    model_run = model_runs[0]
+def test_model_run_data_rows_delete(model_run_with_model_run_data_rows):
+    model_run = model_run_with_model_run_data_rows
 
     before = list(model_run.model_run_data_rows())
     annotation_data_row = before[0]

tests/integration/test_client_errors.py

@@ -45,7 +45,7 @@ def test_semantic_error(client):
 def test_timeout_error(client, project):
     with pytest.raises(labelbox.exceptions.TimeoutError) as excinfo:
         query_str = """query getOntology { 
-        project (where: {id: $%s}) { 
+        project (where: {id: %s}) { 
             ontology { 
                 normalized 
                 } 

tests/integration/test_data_row_metadata.py

@@ -4,6 +4,7 @@
 import uuid
 
 from labelbox import DataRow, Dataset
+from labelbox.exceptions import MalformedQueryException
 from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadata, DataRowMetadataKind, DeleteDataRowMetadata, \
     DataRowMetadataOntology, _parse_metadata_schema
 
@@ -30,9 +31,11 @@
 @pytest.fixture
 def mdo(client):
     mdo = client.get_data_row_metadata_ontology()
-    for schema in mdo.custom_fields:
-        mdo.delete_schema(schema.name)
-    mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string)
+    try:
+        mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string)
+    except MalformedQueryException:
+        # Do nothing if already exists
+        pass
     mdo._raw_ontology = mdo._get_ontology()
     mdo._raw_ontology.append(FAKE_NUMBER_FIELD)
     mdo._build_ontology()
@@ -101,7 +104,8 @@ def test_export_empty_metadata(client, configured_project_with_label,
 def test_get_datarow_metadata_ontology(mdo):
     assert len(mdo.fields)
     assert len(mdo.reserved_fields)
-    assert len(mdo.custom_fields) == 2
+    # two are created by mdo fixture but there may be more
+    assert len(mdo.custom_fields) >= 2
 
     split = mdo.reserved_by_name["split"]["train"]
 

tests/integration/test_data_rows.py

@@ -7,6 +7,7 @@
 import requests
 
 from labelbox import DataRow
+from labelbox.exceptions import MalformedQueryException
 from labelbox.schema.task import Task
 from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadataKind
 import labelbox.exceptions
@@ -26,9 +27,11 @@
 @pytest.fixture
 def mdo(client):
     mdo = client.get_data_row_metadata_ontology()
-    for schema in mdo.custom_fields:
-        mdo.delete_schema(schema.name)
-    mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string)
+    try:
+        mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string)
+    except MalformedQueryException:
+        # Do nothing if already exists
+        pass
     mdo._raw_ontology = mdo._get_ontology()
     mdo._build_ontology()
     yield mdo
@@ -415,7 +418,7 @@ def test_create_data_rows_with_named_metadata_field_class(
             "row1",
         DataRow.metadata_fields: [
             DataRowMetadataField(name='split', value='test'),
-            DataRowMetadataField(name='custom_text', value='hello')
+            DataRowMetadataField(name=CUSTOM_TEXT_SCHEMA_NAME, value='hello')
         ]
     }
 
@@ -430,7 +433,7 @@ def test_create_data_rows_with_named_metadata_field_class(
                 'value': 'test'
             },
             {
-                'name': 'custom_text',
+                'name': CUSTOM_TEXT_SCHEMA_NAME,
                 'value': 'hello'
             },
         ]

tests/integration/test_dataset.py

@@ -7,27 +7,19 @@
 
 
 def test_dataset(client, rand_gen):
-    before = list(client.get_datasets())
-    for o in before:
-        assert isinstance(o, Dataset)
 
+    # confirm dataset can be created
     name = rand_gen(str)
     dataset = client.create_dataset(name=name)
     assert dataset.name == name
     assert dataset.created_by() == client.get_user()
     assert dataset.organization() == client.get_organization()
 
-    after = list(client.get_datasets())
-    assert len(after) == len(before) + 1
-    assert dataset in after
-
-    # confirm get_one returns first dataset
-    get_one_dataset = client.get_datasets().get_one()
-    assert get_one_dataset.uid == after[0].uid
-
-    # confirm get_many(1) returns first dataset
-    get_many_datasets = client.get_datasets().get_many(1)
-    assert get_many_datasets[0].uid == after[0].uid
+    retrieved_dataset = client.get_dataset(dataset.uid)
+    assert retrieved_dataset.name == dataset.name
+    assert retrieved_dataset.uid == dataset.uid
+    assert retrieved_dataset.created_by() == dataset.created_by()
+    assert retrieved_dataset.organization() == dataset.organization()
 
     dataset = client.get_dataset(dataset.uid)
     assert dataset.name == name
@@ -48,9 +40,6 @@ def test_dataset(client, rand_gen):
     assert dataset.description == description
 
     dataset.delete()
-    final = list(client.get_datasets())
-    assert dataset not in final
-    assert set(final) == set(before)
 
     with pytest.raises(ResourceNotFoundError):
         dataset = client.get_dataset(dataset.uid)

tests/integration/test_pagination.py

@@ -0,0 +1,13 @@
+from copy import copy
+
+
+def test_get_one_and_many_dataset_order(client):
+    paginator = client.get_datasets()
+    # confirm get_one returns first dataset
+    all_datasets = list(paginator)
+    get_one_dataset = copy(paginator).get_one()
+    assert get_one_dataset.uid == all_datasets[0].uid
+
+    # confirm get_many(1) returns first dataset
+    get_many_datasets = copy(paginator).get_many(1)
+    assert get_many_datasets[0].uid == all_datasets[0].uid