Add instrumentation for fixtures(temp)

Author: Val Brodsky

tests/integration/annotation_import/conftest.py

@@ -9,6 +9,7 @@
 from typing import Type
 from labelbox.schema.labeling_frontend import LabelingFrontend
 from labelbox.schema.annotation_import import LabelImport, AnnotationImportState
+from labelbox.schema.project import Project
 from labelbox.schema.queue_mode import QueueMode
 
 DATA_ROW_PROCESSING_WAIT_TIMEOUT_SECONDS = 40
@@ -486,6 +487,7 @@ def initial_dataset(client, rand_gen):
 
 @pytest.fixture
 def configured_project(client, initial_dataset, ontology, rand_gen, image_url):
+    start_time = time.time()
     dataset = initial_dataset
     project = client.create_project(
         name=rand_gen(str),
@@ -496,14 +498,21 @@ def configured_project(client, initial_dataset, ontology, rand_gen, image_url):
             where=LabelingFrontend.name == "editor"))[0]
     project.setup(editor, ontology)
     data_row_ids = []
-
+    # print("Before creating data rows ", time.time() - start_time)
+    num_rows = 0
     for _ in range(len(ontology['tools']) + len(ontology['classifications'])):
         data_row_ids.append(dataset.create_data_row(row_data=image_url).uid)
+        num_rows += 1
+    # print("After creating data rows ", time.time() - start_time)
+
+    pytest.data_row_report['times'] += time.time() - start_time
+    pytest.data_row_report['num_rows'] += num_rows
     project.create_batch(
         rand_gen(str),
         data_row_ids,  # sample of data row objects
         5  # priority between 1(Highest) - 5(lowest)
     )
+    print("After creating batch ", time.time() - start_time)
     project.data_row_ids = data_row_ids
     yield project
     project.delete()
@@ -1006,6 +1015,7 @@ def model_run_with_training_metadata(rand_gen, model):
 @pytest.fixture
 def model_run_with_data_rows(client, configured_project, model_run_predictions,
                              model_run, wait_for_label_processing):
+    start_time = time.time()
     configured_project.enable_model_assisted_labeling()
 
     upload_task = LabelImport.create_from_objects(
@@ -1019,6 +1029,7 @@ def model_run_with_data_rows(client, configured_project, model_run_predictions,
     labels = wait_for_label_processing(configured_project)
     label_ids = [label.uid for label in labels]
     model_run.upsert_labels(label_ids)
+    print(f"model_run_with_data_rows: {time.time() - start_time}")
     yield model_run
     model_run.delete()
     # TODO: Delete resources when that is possible ..

tests/integration/conftest.py

@@ -1,3 +1,5 @@
+from collections import defaultdict
+from itertools import islice
 import json
 import os
 import re
@@ -807,3 +809,37 @@ def upload_invalid_data_rows_for_dataset(dataset: Dataset):
         },
     ] * 2)
     task.wait_till_done()
+
+
+def pytest_configure():
+    pytest.report = defaultdict(int)
+    pytest.data_row_report = {'times': 0, 'num_rows': 0}
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_fixture_setup(fixturedef, request):
+    start = time.time()
+    yield
+
+    end = time.time()
+
+    exec_time = end - start
+    pytest.report[fixturedef.argname] += exec_time
+
+    # print('pytest_fixture_setup'
+    #       f', request={request}'
+    #       f', create_data_row_time={end - start}')
+
+
+@pytest.fixture(scope='session', autouse=True)
+def print_perf_summary():
+    yield
+
+    sorted_dict = dict(
+        sorted(pytest.report.items(), key=lambda item: item[1], reverse=True))
+    num_of_entries = 10 if len(sorted_dict) >= 10 else len(sorted_dict)
+    slowest_fixtures = [
+        (aaa, sorted_dict[aaa]) for aaa in islice(sorted_dict, num_of_entries)
+    ]
+    print("\nTop slowest fixtures:\n", slowest_fixtures)
+    print("Data row report:\n", pytest.data_row_report)

tests/integration/test_dataset.py

@@ -53,8 +53,12 @@ def dataset_for_filtering(client, rand_gen):
 
     yield name_1, d1, name_2, d2
 
-    d1.delete()
-    d2.delete()
+
+def test_dataset_filtering(client, dataset_for_filtering):
+    name_1, d1, name_2, d2 = dataset_for_filtering
+
+    assert list(client.get_datasets(where=Dataset.name == name_1)) == [d1]
+    assert list(client.get_datasets(where=Dataset.name == name_2)) == [d2]
 
 
 def test_dataset_filtering(client, dataset_for_filtering):