Skip to content

Commit e404761

Browse files
natoverseAlonsoGuevara
natoverse
and
AlonsoGuevara
authored
Speed up smoke tests (#1736)
* Move verb tests to regular CI * Clean up env vars * Update smoke runtime expectations * Rework artifact assertions * Fix plural in name * remove redundant artifact len check * Remove redundant artifact len check * Adjust graph output expectations * Update community expectations * Include all workflow output * Adjust text unit expectations * Adjust assertions per dataset * Fix test config param name * Update nan allowed for optional model fields --------- Co-authored-by: Alonso Guevara <alonsog@microsoft.com>
1 parent 61a309b commit e404761

File tree

7 files changed

+95
-108
lines changed

7 files changed

+95
-108
lines changed

‎.github/workflows/python-ci.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,3 +87,7 @@ jobs:
8787
- name: Unit Test
8888
run: |
8989
poetry run poe test_unit
90+
91+
- name: Verb Test
92+
run: |
93+
poetry run poe test_verbs

‎.github/workflows/python-smoke-tests.yml‎

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -47,17 +47,13 @@ jobs:
4747
GRAPHRAG_API_VERSION: ${{ secrets.GRAPHRAG_API_VERSION }}
4848
GRAPHRAG_LLM_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_LLM_DEPLOYMENT_NAME }}
4949
GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME }}
50-
GRAPHRAG_CACHE_CONTAINER_NAME: "cicache"
51-
GRAPHRAG_CACHE_BASE_DIR": "cache"
5250
GRAPHRAG_LLM_MODEL: ${{ secrets.GRAPHRAG_LLM_MODEL }}
5351
GRAPHRAG_EMBEDDING_MODEL: ${{ secrets.GRAPHRAG_EMBEDDING_MODEL }}
54-
# We have Windows + Linux runners in 3.10 and 3.11, so we need to divide the rate limits by 4
55-
GRAPHRAG_LLM_TPM: 100_000 # 400_000 / 4
56-
GRAPHRAG_LLM_RPM: 500 # 2_000 / 4
57-
GRAPHRAG_EMBEDDING_TPM: 87_500 # 350,000 / 4
58-
GRAPHRAG_EMBEDDING_RPM: 525 # 2,100 / 4
59-
GRAPHRAG_CHUNK_SIZE: 1200
60-
GRAPHRAG_CHUNK_OVERLAP: 0
52+
# We have Windows + Linux runners in 3.10, so we need to divide the rate limits by 2
53+
GRAPHRAG_LLM_TPM: 200_000 # 400_000 / 2
54+
GRAPHRAG_LLM_RPM: 1_000 # 2_000 / 2
55+
GRAPHRAG_EMBEDDING_TPM: 225_000 # 450_000 / 2
56+
GRAPHRAG_EMBEDDING_RPM: 1_000 # 2_000 / 2
6157
# Azure AI Search config
6258
AZURE_AI_SEARCH_URL_ENDPOINT: ${{ secrets.AZURE_AI_SEARCH_URL_ENDPOINT }}
6359
AZURE_AI_SEARCH_API_KEY: ${{ secrets.AZURE_AI_SEARCH_API_KEY }}
@@ -101,10 +97,6 @@ jobs:
10197
run: |
10298
poetry build
10399
104-
- name: Verb Test
105-
run: |
106-
poetry run poe test_verbs
107-
108100
- name: Install Azurite
109101
id: azuright
110102
uses: potatoqualitee/azuright@v1.1
@@ -118,4 +110,4 @@ jobs:
118110
if: always()
119111
with:
120112
name: smoke-test-artifacts-${{ matrix.python-version }}-${{ matrix.poetry-version }}-${{ runner.os }}
121-
path: tests/fixtures/*/output
113+
path: tests/fixtures/*

‎tests/fixtures/min-csv/config.json‎

Lines changed: 37 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -3,41 +3,38 @@
33
"input_file_type": "text",
44
"workflow_config": {
55
"create_base_text_units": {
6-
"row_range": [
7-
1,
8-
2500
9-
],
10-
"max_runtime": 150,
11-
"expected_artifacts": 0
6+
"max_runtime": 30
127
},
138
"extract_graph": {
14-
"row_range": [
15-
1,
16-
2500
17-
],
18-
"max_runtime": 500,
19-
"expected_artifacts": 0
9+
"max_runtime": 500
2010
},
2111
"finalize_graph": {
2212
"row_range": [
2313
1,
24-
2500
14+
500
15+
],
16+
"nan_allowed_columns": [
17+
"x",
18+
"y"
2519
],
2620
"max_runtime": 30,
27-
"expected_artifacts": 2
21+
"expected_artifacts": [
22+
"entities.parquet",
23+
"relationships.parquet"
24+
]
2825
},
2926
"create_communities": {
3027
"row_range": [
31-
1,
32-
2500
28+
10,
29+
30
3330
],
34-
"max_runtime": 150,
35-
"expected_artifacts": 1
31+
"max_runtime": 30,
32+
"expected_artifacts": ["communities.parquet"]
3633
},
3734
"create_community_reports": {
3835
"row_range": [
39-
1,
40-
2500
36+
10,
37+
30
4138
],
4239
"nan_allowed_columns": [
4340
"title",
@@ -51,35 +48,43 @@
5148
"size"
5249
],
5350
"max_runtime": 300,
54-
"expected_artifacts": 1
51+
"expected_artifacts": ["community_reports.parquet"]
5552
},
5653
"create_final_text_units": {
5754
"row_range": [
58-
1,
59-
2500
55+
10,
56+
50
6057
],
6158
"nan_allowed_columns": [
6259
"relationship_ids",
63-
"entity_ids"
60+
"entity_ids",
61+
"covariate_ids"
6462
],
65-
"max_runtime": 150,
66-
"expected_artifacts": 1
63+
"max_runtime": 30,
64+
"expected_artifacts": ["text_units.parquet"]
6765
},
6866
"create_final_documents": {
6967
"row_range": [
70-
1,
71-
2500
68+
15,
69+
15
7270
],
73-
"max_runtime": 150,
74-
"expected_artifacts": 1
71+
"nan_allowed_columns": [
72+
"metadata"
73+
],
74+
"max_runtime": 30,
75+
"expected_artifacts": ["documents.parquet"]
7576
},
7677
"generate_text_embeddings": {
7778
"row_range": [
7879
1,
79-
2500
80+
500
8081
],
8182
"max_runtime": 150,
82-
"expected_artifacts": 1
83+
"expected_artifacts": [
84+
"embeddings.text_unit.text.parquet",
85+
"embeddings.entity.description.parquet",
86+
"embeddings.community.full_content.parquet"
87+
]
8388
}
8489
},
8590
"query_config": [

‎tests/fixtures/min-csv/settings.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,5 +41,5 @@ snapshots:
4141

4242
drift_search:
4343
n_depth: 1
44-
k_follow_ups: 3
44+
drift_k_followups: 3
4545
primer_folds: 3

‎tests/fixtures/text/config.json‎

Lines changed: 35 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -3,33 +3,30 @@
33
"input_file_type": "text",
44
"workflow_config": {
55
"create_base_text_units": {
6-
"row_range": [
7-
1,
8-
2500
9-
],
10-
"max_runtime": 150,
11-
"expected_artifacts": 0
6+
"max_runtime": 30
127
},
138
"extract_graph": {
14-
"row_range": [
15-
1,
16-
2500
17-
],
18-
"max_runtime": 500,
19-
"expected_artifacts": 0
9+
"max_runtime": 500
2010
},
2111
"finalize_graph": {
2212
"row_range": [
2313
1,
24-
2500
14+
100
15+
],
16+
"nan_allowed_columns": [
17+
"x",
18+
"y"
2519
],
2620
"max_runtime": 30,
27-
"expected_artifacts": 2
21+
"expected_artifacts": [
22+
"entities.parquet",
23+
"relationships.parquet"
24+
]
2825
},
2926
"extract_covariates": {
3027
"row_range": [
3128
1,
32-
2500
29+
100
3330
],
3431
"nan_allowed_columns": [
3532
"type",
@@ -41,20 +38,20 @@
4138
"source_text"
4239
],
4340
"max_runtime": 300,
44-
"expected_artifacts": 1
41+
"expected_artifacts": ["covariates.parquet"]
4542
},
4643
"create_communities": {
4744
"row_range": [
4845
1,
49-
2500
46+
30
5047
],
51-
"max_runtime": 150,
52-
"expected_artifacts": 1
48+
"max_runtime": 30,
49+
"expected_artifacts": ["communities.parquet"]
5350
},
5451
"create_community_reports": {
5552
"row_range": [
5653
1,
57-
2500
54+
30
5855
],
5956
"nan_allowed_columns": [
6057
"title",
@@ -68,35 +65,43 @@
6865
"size"
6966
],
7067
"max_runtime": 300,
71-
"expected_artifacts": 1
68+
"expected_artifacts": ["community_reports.parquet"]
7269
},
7370
"create_final_text_units": {
7471
"row_range": [
7572
1,
76-
2500
73+
10
7774
],
7875
"nan_allowed_columns": [
7976
"relationship_ids",
80-
"entity_ids"
77+
"entity_ids",
78+
"covariate_ids"
8179
],
82-
"max_runtime": 150,
83-
"expected_artifacts": 1
80+
"max_runtime": 30,
81+
"expected_artifacts": ["text_units.parquet"]
8482
},
8583
"create_final_documents": {
8684
"row_range": [
8785
1,
88-
2500
86+
1
8987
],
90-
"max_runtime": 150,
91-
"expected_artifacts": 1
88+
"nan_allowed_columns": [
89+
"metadata"
90+
],
91+
"max_runtime": 30,
92+
"expected_artifacts": ["documents.parquet"]
9293
},
9394
"generate_text_embeddings": {
9495
"row_range": [
9596
1,
96-
2500
97+
100
9798
],
9899
"max_runtime": 150,
99-
"expected_artifacts": 1
100+
"expected_artifacts": [
101+
"embeddings.text_unit.text.parquet",
102+
"embeddings.entity.description.parquet",
103+
"embeddings.community.full_content.parquet"
104+
]
100105
}
101106
},
102107
"query_config": [

‎tests/fixtures/text/settings.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,5 +45,5 @@ snapshots:
4545

4646
drift_search:
4747
n_depth: 1
48-
k_follow_ups: 3
48+
drift_k_followups: 3
4949
primer_folds: 3

‎tests/smoke/test_fixtures.py‎

Lines changed: 11 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -160,64 +160,45 @@ def __assert_indexer_outputs(
160160
stats = json.loads((output_path / "stats.json").read_bytes().decode("utf-8"))
161161

162162
# Check all workflows run
163-
expected_artifacts = 0
164163
expected_workflows = set(workflow_config.keys())
165164
workflows = set(stats["workflows"].keys())
166165
assert workflows == expected_workflows, (
167166
f"Workflows missing from stats.json: {expected_workflows - workflows}. Unexpected workflows in stats.json: {workflows - expected_workflows}"
168167
)
169168

170169
# [OPTIONAL] Check runtime
171-
for workflow in expected_workflows:
170+
for workflow, config in workflow_config.items():
172171
# Check expected artifacts
173-
expected_artifacts = expected_artifacts + workflow_config[workflow].get(
174-
"expected_artifacts", 1
175-
)
172+
workflow_artifacts = config.get("expected_artifacts", [])
176173
# Check max runtime
177-
max_runtime = workflow_config[workflow].get("max_runtime", None)
174+
max_runtime = config.get("max_runtime", None)
178175
if max_runtime:
179176
assert stats["workflows"][workflow]["overall"] <= max_runtime, (
180177
f"Expected max runtime of {max_runtime}, found: {stats['workflows'][workflow]['overall']} for workflow: {workflow}"
181178
)
182-
183-
# Check artifacts
184-
artifact_files = os.listdir(output_path)
185-
186-
# check that the number of workflows matches the number of artifacts
187-
assert len(artifact_files) == (expected_artifacts + 3), (
188-
f"Expected {expected_artifacts + 3} artifacts, found: {len(artifact_files)}"
189-
) # Embeddings add to the count
190-
191-
for artifact in artifact_files:
192-
if artifact.endswith(".parquet"):
193-
output_df = pd.read_parquet(output_path / artifact)
194-
artifact_name = artifact.split(".")[0]
195-
196-
try:
197-
workflow = workflow_config[artifact_name]
179+
# Check expected artifacts
180+
for artifact in workflow_artifacts:
181+
if artifact.endswith(".parquet"):
182+
output_df = pd.read_parquet(output_path / artifact)
198183

199184
# Check number of rows between range
200185
assert (
201-
workflow["row_range"][0]
186+
config["row_range"][0]
202187
<= len(output_df)
203-
<= workflow["row_range"][1]
188+
<= config["row_range"][1]
204189
), (
205-
f"Expected between {workflow['row_range'][0]} and {workflow['row_range'][1]}, found: {len(output_df)} for file: {artifact}"
190+
f"Expected between {config['row_range'][0]} and {config['row_range'][1]}, found: {len(output_df)} for file: {artifact}"
206191
)
207192

208193
# Get non-nan rows
209194
nan_df = output_df.loc[
210195
:,
211-
~output_df.columns.isin(
212-
workflow.get("nan_allowed_columns", [])
213-
),
196+
~output_df.columns.isin(config.get("nan_allowed_columns", [])),
214197
]
215198
nan_df = nan_df[nan_df.isna().any(axis=1)]
216199
assert len(nan_df) == 0, (
217200
f"Found {len(nan_df)} rows with NaN values for file: {artifact} on columns: {nan_df.columns[nan_df.isna().any()].tolist()}"
218201
)
219-
except KeyError:
220-
log.warning("No workflow config found %s", artifact_name)
221202

222203
def __run_query(self, root: Path, query_config: dict[str, str]):
223204
command = [

0 commit comments

Comments
 (0)