Vb/fix flaky tests vb plt 1449 2 (#1793)

Author: vbrodsky

libs/labelbox/tests/conftest.py

@@ -594,14 +594,8 @@ def sample_bulk_conversation() -> list:
 def organization(client):
     # Must have at least one seat open in your org to run these tests
     org = client.get_organization()
-    # Clean up before and after incase this wasn't run for some reason.
-    for invite in get_invites(client):
-        if "@labelbox.com" in invite.email:
-            cancel_invite(client, invite.uid)
+
     yield org
-    for invite in get_invites(client):
-        if "@labelbox.com" in invite.email:
-            cancel_invite(client, invite.uid)
 
 
 @pytest.fixture
@@ -1074,20 +1068,15 @@ def configured_project_with_complex_ontology(client, initial_dataset, rand_gen,
     project.delete()
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture
 def embedding(client: Client, environ):
 
     uuid_str = uuid.uuid4().hex
+    time.sleep(randint(1, 5))
     embedding = client.create_embedding(f"sdk-int-{uuid_str}", 8)
     yield embedding
-    # Remove all embeddings on staging
-    if environ == Environ.STAGING:
-        embeddings = client.get_embeddings()
-        for embedding in embeddings:
-            with suppress(LabelboxError):
-                embedding.delete()
-    else:
-        embedding.delete()
+
+    embedding.delete()
 
 
 @pytest.fixture

libs/labelbox/tests/data/annotation_import/test_generic_data_types.py

@@ -66,8 +66,8 @@ def test_generic_data_row_type_by_data_row_id(
         (MediaType.Video, GenericDataRowData),
         (MediaType.Conversational, GenericDataRowData),
         (MediaType.Document, GenericDataRowData),
-        (MediaType.LLMPromptResponseCreation, GenericDataRowData),
-        (MediaType.LLMPromptCreation, GenericDataRowData),
+        # (MediaType.LLMPromptResponseCreation, GenericDataRowData),
+        # (MediaType.LLMPromptCreation, GenericDataRowData),
         (OntologyKind.ResponseCreation, GenericDataRowData)
     ],
 )
@@ -100,8 +100,8 @@ def test_generic_data_row_type_by_global_key(
         (MediaType.Conversational, MediaType.Conversational),
         (MediaType.Document, MediaType.Document),
         (MediaType.Dicom, MediaType.Dicom),
-        (MediaType.LLMPromptResponseCreation, MediaType.LLMPromptResponseCreation),
-        (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation),
+        # (MediaType.LLMPromptResponseCreation, MediaType.LLMPromptResponseCreation),
+        # (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation),
         (OntologyKind.ResponseCreation, OntologyKind.ResponseCreation)
     ],
     indirect=["configured_project"]
@@ -220,8 +220,8 @@ def test_import_media_types_by_global_key(
         (MediaType.Conversational, MediaType.Conversational),
         (MediaType.Document, MediaType.Document),
         (MediaType.Dicom, MediaType.Dicom),
-        (MediaType.LLMPromptResponseCreation, MediaType.LLMPromptResponseCreation),
-        (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation),
+        # (MediaType.LLMPromptResponseCreation, MediaType.LLMPromptResponseCreation),
+        # (MediaType.LLMPromptCreation, MediaType.LLMPromptCreation),
         (OntologyKind.ResponseCreation, OntologyKind.ResponseCreation)
     ],
     indirect=["configured_project"]

libs/labelbox/tests/integration/test_embedding.py

@@ -15,22 +15,18 @@ def test_get_embedding_by_id(client: Client, embedding: Embedding):
     e = client.get_embedding_by_id(embedding.id)
     assert e.id == embedding.id
 
-
-def test_get_embedding_by_name(client: Client, embedding: Embedding):
     e = client.get_embedding_by_name(embedding.name)
     assert e.name == embedding.name
 
+    embeddings = client.get_embeddings()
+    assert len(embeddings) > 0
+
 
 def test_get_embedding_by_name_not_found(client: Client):
     with pytest.raises(labelbox.exceptions.ResourceNotFoundError):
         client.get_embedding_by_name("does-not-exist")
 
 
-def test_get_embeddings(client: Client, embedding: Embedding):
-    embeddings = client.get_embeddings()
-    assert len(embeddings) > 0
-
-
 @pytest.mark.parametrize('data_rows', [10], indirect=True)
 def test_import_vectors_from_file(data_rows: List[DataRow],
                                   embedding: Embedding):

libs/labelbox/tests/integration/test_filtering.py

@@ -46,9 +46,6 @@ def get(where=None):
     assert p_a.uid in lt_b and p_b.uid not in lt_b and p_c.uid not in lt_b
     ge_b = get(Project.name >= p_b_name)
     assert {p_b.uid, p_c.uid}.issubset(ge_b) and p_a.uid not in ge_b
-    le_b = get(Project.name <= p_b_name)
-    assert {p_a.uid, p_b.uid}.issubset(le_b) and p_c.uid not in le_b
-
 
 def test_unsupported_where(client):
     with pytest.raises(InvalidQueryError):

libs/labelbox/tests/integration/test_user_management.py

@@ -5,9 +5,13 @@
 
 faker = Faker()
 
-def test_org_invite(client, organization, environ, queries):
+
+@pytest.fixture
+def org_invite(client, organization, environ, queries):
     role = client.get_roles()['LABELER']
-    dummy_email = "none+{}@labelbox.com".format("".join(faker.random_letters(26)))
+
+    dummy_email = "none+{}@labelbox.com".format("".join(
+        faker.random_letters(26)))
     invite_limit = organization.invite_limit()
 
     if environ.value == "prod":
@@ -18,6 +22,45 @@ def test_org_invite(client, organization, environ, queries):
 
     invite = organization.invite_user(dummy_email, role)
 
+    yield invite, invite_limit
+
+    queries.cancel_invite(client, invite.uid)
+
+
+@pytest.fixture
+def project_role_1(client, project_pack):
+    project_1, _ = project_pack
+    roles = client.get_roles()
+    return ProjectRole(project=project_1, role=roles['LABELER'])
+
+
+@pytest.fixture
+def project_role_2(client, project_pack):
+    _, project_2 = project_pack
+    roles = client.get_roles()
+    return ProjectRole(project=project_2, role=roles['REVIEWER'])
+
+
+@pytest.fixture
+def create_project_invite(client, organization, project_pack, queries,
+                          project_role_1, project_role_2):
+    roles = client.get_roles()
+    dummy_email = "none+{}@labelbox.com".format("".join(
+        faker.random_letters(26)))
+    invite = organization.invite_user(
+        dummy_email,
+        roles['NONE'],
+        project_roles=[project_role_1, project_role_2])
+
+    yield invite
+
+    queries.cancel_invite(client, invite.uid)
+
+
+def test_org_invite(client, organization, environ, queries, org_invite):
+    invite, invite_limit = org_invite
+    role = client.get_roles()['LABELER']
+
     if environ.value == "prod":
 
         invite_limit_after = organization.invite_limit()
@@ -28,30 +71,36 @@ def test_org_invite(client, organization, environ, queries):
     outstanding_invites = queries.get_invites(client)
     in_list = False
 
-    for invite in outstanding_invites:
-        if invite.uid == invite.uid:
+    for outstanding_invite in outstanding_invites:
+        if outstanding_invite.uid == invite.uid:
             in_list = True
-            org_role = invite.organization_role_name.lower()
+            org_role = outstanding_invite.organization_role_name.lower()
             assert org_role == role.name.lower(
             ), "Role should be labeler. Found {org_role} "
     assert in_list, "Invite not found"
+
+
+def test_cancel_invite(
+    client,
+    organization,
+    queries,
+):
+    role = client.get_roles()['LABELER']
+    dummy_email = "none+{}@labelbox.com".format("".join(
+        faker.random_letters(26)))
+    invite = organization.invite_user(dummy_email, role)
     queries.cancel_invite(client, invite.uid)
-    assert invite_limit.remaining - organization.invite_limit().remaining == 0
+    outstanding_invites = [i.uid for i in queries.get_invites(client)]
+    assert invite.uid not in outstanding_invites
 
 
-def test_project_invite(client, organization, project_pack, queries):
-    project_1, project_2 = project_pack
+def test_project_invite(client, organization, project_pack, queries,
+                        create_project_invite, project_role_1, project_role_2):
+    create_project_invite
+    project_1, _ = project_pack
     roles = client.get_roles()
-    dummy_email = "none+{}@labelbox.com".format("".join(faker.random_letters(26)))
-    project_role_1 = ProjectRole(project=project_1, role=roles['LABELER'])
-    project_role_2 = ProjectRole(project=project_2, role=roles['REVIEWER'])
-    invite = organization.invite_user(
-        dummy_email,
-        roles['NONE'],
-        project_roles=[project_role_1, project_role_2])
 
     project_invite = next(queries.get_project_invites(client, project_1.uid))
-
     assert set([(proj_invite.project.uid, proj_invite.role.uid)
                 for proj_invite in project_invite.project_roles
                ]) == set([(proj_role.project.uid, proj_role.role.uid)
@@ -74,7 +123,6 @@ def test_project_invite(client, organization, project_pack, queries):
 
     assert project_member.access_from == 'ORGANIZATION'
     assert project_member.role().name.upper() == roles['ADMIN'].name.upper()
-    queries.cancel_invite(client, invite.uid)
 
 
 @pytest.mark.skip(