Further test parallelization

Author: whistler

pytest.ini

@@ -1,4 +1,4 @@
 [pytest]
-addopts = -s -vv -x --reruns 5 --reruns-delay 10 --durations=20
+addopts = -s -vv --reruns 5 --reruns-delay 10 --durations=20
 markers =
     slow: marks tests as slow (deselect with '-m "not slow"')

requirements.txt

@@ -13,4 +13,5 @@ typeguard
 imagesize
 pyproj
 pygeotile
-typing-extensions
+typing-extensions
+pytest-xdist

tests/integration/annotation_import/test_model.py

@@ -1,31 +1,26 @@
+import pytest
+
 from labelbox import Model
+from labelbox.exceptions import ResourceNotFoundError
 
 
 def test_model(client, configured_project, rand_gen):
-    before = list(client.get_models())
-    for m in before:
+    # Get all
+    models = list(client.get_models())
+    for m in models:
         assert isinstance(m, Model)
 
+    # Create
     ontology = configured_project.ontology()
-
     data = {"name": rand_gen(str), "ontology_id": ontology.uid}
     model = client.create_model(data["name"], data["ontology_id"])
     assert model.name == data["name"]
 
-    after = list(client.get_models())
-    assert len(after) == len(before) + 1
-    assert model in after
-
+    # Get one
     model = client.get_model(model.uid)
     assert model.name == data["name"]
 
-
-def test_model_delete(client, model):
-    before = list(client.get_models())
-
-    model = before[0]
+    # Delete
     model.delete()
-
-    after = list(client.get_models())
-
-    assert len(before) == len(after) + 1
+    with pytest.raises(ResourceNotFoundError):
+        client.get_model(model.uid)

tests/integration/annotation_import/test_model_run.py

@@ -52,8 +52,9 @@ def test_model_run_delete(client, model_run):
     models_after = list(client.get_models())
     model_after = models_after[0]
     after = list(model_after.model_runs())
+    after_uids = {mr.uid for mr in after}
 
-    assert len(before) == len(after) + 1
+    assert model_run.uid not in after_uids
 
 
 def test_model_run_update_config(model_run_with_training_metadata):
@@ -75,10 +76,11 @@ def test_model_run_get_config(model_run_with_training_metadata):
 
 
 def test_model_run_data_rows_delete(client, model_run_with_model_run_data_rows):
-    models = list(client.get_models())
-    model = models[0]
-    model_runs = list(model.model_runs())
-    model_run = model_runs[0]
+    # models = list(client.get_models())
+    # model = models[0]
+    # model_runs = list(model.model_runs())
+    # model_run = model_runs[0]
+    model_run = model_run_with_model_run_data_rows
 
     before = list(model_run.model_run_data_rows())
     annotation_data_row = before[0]

tests/integration/test_client_errors.py

@@ -45,7 +45,7 @@ def test_semantic_error(client):
 def test_timeout_error(client, project):
     with pytest.raises(labelbox.exceptions.TimeoutError) as excinfo:
         query_str = """query getOntology { 
-        project (where: {id: $%s}) { 
+        project (where: {id: %s}) { 
             ontology { 
                 normalized 
                 } 

tests/integration/test_data_row_metadata.py

@@ -4,6 +4,7 @@
 import uuid
 
 from labelbox import DataRow, Dataset
+from labelbox.exceptions import MalformedQueryException
 from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadata, DataRowMetadataKind, DeleteDataRowMetadata, \
     DataRowMetadataOntology, _parse_metadata_schema
 
@@ -17,7 +18,7 @@
 TEXT_SCHEMA_ID = "cko8s9r5v0001h2dk9elqdidh"
 CAPTURE_DT_SCHEMA_ID = "cko8sdzv70006h2dk8jg64zvb"
 PRE_COMPUTED_EMBEDDINGS_ID = 'ckrzang79000008l6hb5s6za1'
-CUSTOM_TEXT_SCHEMA_NAME = 'custom_text'
+CUSTOM_TEXT_SCHEMA_NAME = 'datarow_metadata_custom_text'
 
 FAKE_NUMBER_FIELD = {
     "id": FAKE_SCHEMA_ID,
@@ -30,13 +31,16 @@
 @pytest.fixture
 def mdo(client):
     mdo = client.get_data_row_metadata_ontology()
-    for schema in mdo.custom_fields:
-        mdo.delete_schema(schema.name)
-    mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string)
+    try:
+        mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string)
+    except MalformedQueryException:
+        # Do nothing if already exists
+        pass
     mdo._raw_ontology = mdo._get_ontology()
     mdo._raw_ontology.append(FAKE_NUMBER_FIELD)
     mdo._build_ontology()
     yield mdo
+    mdo.delete_schema(CUSTOM_TEXT_SCHEMA_NAME)
 
 
 @pytest.fixture
@@ -101,7 +105,8 @@ def test_export_empty_metadata(client, configured_project_with_label,
 def test_get_datarow_metadata_ontology(mdo):
     assert len(mdo.fields)
     assert len(mdo.reserved_fields)
-    assert len(mdo.custom_fields) == 2
+    # three are created by mdo fixture but there may be more
+    assert len(mdo.custom_fields) >= 3
 
     split = mdo.reserved_by_name["split"]["train"]
 

tests/integration/test_data_rows.py

@@ -7,6 +7,7 @@
 import requests
 
 from labelbox import DataRow
+from labelbox.exceptions import MalformedQueryException
 from labelbox.schema.task import Task
 from labelbox.schema.data_row_metadata import DataRowMetadataField, DataRowMetadataKind
 import labelbox.exceptions
@@ -20,18 +21,21 @@
     SPLIT_SCHEMA_ID, TEST_SPLIT_ID, EMBEDDING_SCHEMA_ID, TEXT_SCHEMA_ID,
     CAPTURE_DT_SCHEMA_ID
 ].sort()
-CUSTOM_TEXT_SCHEMA_NAME = "custom_text"
+CUSTOM_TEXT_SCHEMA_NAME = "data_row_custom_text"
 
 
 @pytest.fixture
 def mdo(client):
     mdo = client.get_data_row_metadata_ontology()
-    for schema in mdo.custom_fields:
-        mdo.delete_schema(schema.name)
-    mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string)
+    try:
+        mdo.create_schema(CUSTOM_TEXT_SCHEMA_NAME, DataRowMetadataKind.string)
+    except MalformedQueryException:
+        # Do nothing if already exists
+        pass
     mdo._raw_ontology = mdo._get_ontology()
     mdo._build_ontology()
     yield mdo
+    mdo.delete_schema(CUSTOM_TEXT_SCHEMA_NAME)
 
 
 @pytest.fixture
@@ -415,7 +419,7 @@ def test_create_data_rows_with_named_metadata_field_class(
             "row1",
         DataRow.metadata_fields: [
             DataRowMetadataField(name='split', value='test'),
-            DataRowMetadataField(name='custom_text', value='hello')
+            DataRowMetadataField(name=CUSTOM_TEXT_SCHEMA_NAME, value='hello')
         ]
     }
 
@@ -430,7 +434,7 @@ def test_create_data_rows_with_named_metadata_field_class(
                 'value': 'test'
             },
             {
-                'name': 'custom_text',
+                'name': CUSTOM_TEXT_SCHEMA_NAME,
                 'value': 'hello'
             },
         ]

tests/integration/test_project.py

@@ -11,9 +11,6 @@
 
 
 def test_project(client, rand_gen):
-    before = list(client.get_projects())
-    for o in before:
-        assert isinstance(o, Project)
 
     data = {
         "name": rand_gen(str),
@@ -24,10 +21,6 @@ def test_project(client, rand_gen):
     assert project.name == data["name"]
     assert project.description == data["description"]
 
-    after = list(client.get_projects())
-    assert len(after) == len(before) + 1
-    assert project in after
-
     project = client.get_project(project.uid)
     assert project.name == data["name"]
     assert project.description == data["description"]
@@ -44,9 +37,8 @@ def test_project(client, rand_gen):
     assert project.description == update_data["description"]
 
     project.delete()
-    final = list(client.get_projects())
-    assert project not in final
-    assert set(final) == set(before)
+    projects = list(client.get_projects())
+    assert project not in projects
 
 
 def test_update_project_resource_tags(client, rand_gen):

tests/integration/test_user_and_org.py

@@ -19,5 +19,5 @@ def test_user_and_org_projects(project):
 
     assert project.created_by() == user
     assert project.organization() == org
-    assert set(user.projects()) == user_projects.union({project})
-    assert set(org.projects()) == org_projects.union({project})
+    assert project in user_projects
+    assert project in org_projects