Skip to content

Commit f5b4d2f

Browse files
natoversenatoverse
authored
Ci streamline (#988)
* Remove excess vars from gh-pages build * Delete redundant javascript ci * Pull apart testing CI * Clean up integration tests build * Move storage tests to integration CI * Take py 3.10 out of smoke tests matrix * Use minimum supported python version for most tests * Re-run main CI on any test change * Add Josh and Kenny to author list * Update auto-resolve perms
1 parent 98cabba commit f5b4d2f

13 files changed

+85
-97
lines changed

‎.github/workflows/gh-pages.yml‎

Lines changed: 0 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -17,29 +17,6 @@ jobs:
1717
env:
1818
GH_PAGES: 1
1919
DEBUG: 1
20-
GRAPHRAG_LLM_TYPE: "azure_openai_chat"
21-
GRAPHRAG_EMBEDDING_TYPE: "azure_openai_embedding"
22-
GRAPHRAG_API_KEY: ${{ secrets.OPENAI_API_KEY }}
23-
GRAPHRAG_API_BASE: ${{ secrets.GRAPHRAG_API_BASE }}
24-
GRAPHRAG_API_VERSION: ${{ secrets.GRAPHRAG_API_VERSION }}
25-
GRAPHRAG_LLM_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_LLM_DEPLOYMENT_NAME }}
26-
GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME }}
27-
GRAPHRAG_CACHE_TYPE: "blob"
28-
GRAPHRAG_CACHE_CONNECTION_STRING: ${{ secrets.BLOB_STORAGE_CONNECTION_STRING }}
29-
GRAPHRAG_CACHE_CONTAINER_NAME: "cicache"
30-
GRAPHRAG_CACHE_BASE_DIR": "cache"
31-
GRAPHRAG_LLM_MODEL: gpt-3.5-turbo-16k
32-
GRAPHRAG_EMBEDDING_MODEL: text-embedding-ada-002
33-
# We have Windows + Linux runners in 3.10 and 3.11, so we need to divide the rate limits by 4
34-
GRAPHRAG_LLM_TPM: 45_000 # 180,000 / 4
35-
GRAPHRAG_LLM_RPM: 270 # 1,080 / 4
36-
GRAPHRAG_EMBEDDING_TPM: 87_500 # 350,000 / 4
37-
GRAPHRAG_EMBEDDING_RPM: 525 # 2,100 / 4
38-
GRAPHRAG_CHUNK_SIZE: 1200
39-
GRAPHRAG_CHUNK_OVERLAP: 0
40-
# Azure AI Search config
41-
AZURE_AI_SEARCH_URL_ENDPOINT: ${{ secrets.AZURE_AI_SEARCH_URL_ENDPOINT }}
42-
AZURE_AI_SEARCH_API_KEY: ${{ secrets.AZURE_AI_SEARCH_API_KEY }}
4320

4421
steps:
4522
- uses: actions/checkout@v4

‎.github/workflows/issues-autoresolve.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ on:
55

66
permissions:
77
actions: write
8+
issues: write
9+
pull-requests: write
810

911
jobs:
1012
close-issues:

‎.github/workflows/javascript-ci.yml‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎.github/workflows/python-ci.yml‎

Lines changed: 2 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -26,28 +26,6 @@ jobs:
2626
fail-fast: false # Continue running all jobs even if one fails
2727
env:
2828
DEBUG: 1
29-
GRAPHRAG_LLM_TYPE: "azure_openai_chat"
30-
GRAPHRAG_EMBEDDING_TYPE: "azure_openai_embedding"
31-
GRAPHRAG_API_KEY: ${{ secrets.OPENAI_API_KEY }}
32-
GRAPHRAG_API_BASE: ${{ secrets.GRAPHRAG_API_BASE }}
33-
GRAPHRAG_API_VERSION: ${{ secrets.GRAPHRAG_API_VERSION }}
34-
GRAPHRAG_LLM_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_LLM_DEPLOYMENT_NAME }}
35-
GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME: ${{ secrets.GRAPHRAG_EMBEDDING_DEPLOYMENT_NAME }}
36-
GRAPHRAG_CACHE_CONTAINER_NAME: "cicache"
37-
GRAPHRAG_CACHE_BASE_DIR": "cache"
38-
GRAPHRAG_LLM_MODEL: ${{ secrets.GRAPHRAG_LLM_MODEL }}
39-
GRAPHRAG_EMBEDDING_MODEL: ${{ secrets.GRAPHRAG_EMBEDDING_MODEL }}
40-
GRAPHRAG_ENTITY_EXTRACTION_ENCODING_MODEL: ${{ secrets.GRAPHRAG_ENTITY_EXTRACTION_ENCODING_MODEL }}
41-
# We have Windows + Linux runners in 3.10 and 3.11, so we need to divide the rate limits by 4
42-
GRAPHRAG_LLM_TPM: 45_000 # 180,000 / 4
43-
GRAPHRAG_LLM_RPM: 270 # 1,080 / 4
44-
GRAPHRAG_EMBEDDING_TPM: 87_500 # 350,000 / 4
45-
GRAPHRAG_EMBEDDING_RPM: 525 # 2,100 / 4
46-
GRAPHRAG_CHUNK_SIZE: 1200
47-
GRAPHRAG_CHUNK_OVERLAP: 0
48-
# Azure AI Search config
49-
AZURE_AI_SEARCH_URL_ENDPOINT: ${{ secrets.AZURE_AI_SEARCH_URL_ENDPOINT }}
50-
AZURE_AI_SEARCH_API_KEY: ${{ secrets.AZURE_AI_SEARCH_API_KEY }}
5129

5230
runs-on: ${{ matrix.os }}
5331
steps:
@@ -65,7 +43,7 @@ jobs:
6543
- '**/*.toml'
6644
- '**/*.ipynb'
6745
- '.github/workflows/python*.yml'
68-
- 'tests/smoke/*'
46+
- 'tests/**/*'
6947
7048
- name: Set up Python ${{ matrix.python-version }}
7149
uses: actions/setup-python@v5
@@ -92,14 +70,6 @@ jobs:
9270
run: |
9371
poetry build
9472
95-
- name: Install Azurite
96-
id: azuright
97-
uses: potatoqualitee/azuright@v1.1
98-
9973
- name: Unit Test
10074
run: |
101-
poetry run poe test_unit
102-
103-
- name: Integration Test
104-
run: |
105-
poetry run poe test_integration
75+
poetry run poe test_unit

‎.github/workflows/python-integration-tests.yml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
name: Python Integration Tests
2+
on:
3+
push:
4+
branches: [main]
5+
pull_request:
6+
branches: [main]
7+
8+
permissions:
9+
contents: read
10+
pull-requests: read
11+
12+
concurrency:
13+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
14+
# Only run the for the latest commit
15+
cancel-in-progress: true
16+
17+
env:
18+
POETRY_VERSION: 1.8.3
19+
20+
jobs:
21+
python-ci:
22+
strategy:
23+
matrix:
24+
python-version: ["3.10"]
25+
os: [ubuntu-latest, windows-latest]
26+
fail-fast: false # Continue running all jobs even if one fails
27+
env:
28+
DEBUG: 1
29+
30+
runs-on: ${{ matrix.os }}
31+
steps:
32+
- uses: actions/checkout@v4
33+
34+
- uses: dorny/paths-filter@v3
35+
id: changes
36+
with:
37+
filters: |
38+
python:
39+
- 'graphrag/**/*'
40+
- 'poetry.lock'
41+
- 'pyproject.toml'
42+
- '**/*.py'
43+
- '**/*.toml'
44+
- '**/*.ipynb'
45+
- '.github/workflows/python*.yml'
46+
- 'tests/integration/**/*'
47+
48+
- name: Set up Python ${{ matrix.python-version }}
49+
uses: actions/setup-python@v5
50+
with:
51+
python-version: ${{ matrix.python-version }}
52+
53+
- name: Install Poetry
54+
uses: abatilo/actions-poetry@v3.0.0
55+
with:
56+
poetry-version: $POETRY_VERSION
57+
58+
- name: Install dependencies
59+
shell: bash
60+
run: |
61+
poetry self add setuptools wheel
62+
poetry run python -m pip install gensim
63+
poetry install
64+
65+
- name: Build
66+
run: |
67+
poetry build
68+
69+
- name: Install Azurite
70+
id: azuright
71+
uses: potatoqualitee/azuright@v1.1
72+
73+
- name: Integration Test
74+
run: |
75+
poetry run poe test_integration

‎.github/workflows/python-notebook-tests.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
python-ci:
2222
strategy:
2323
matrix:
24-
python-version: ["3.11"]
24+
python-version: ["3.10"]
2525
os: [ubuntu-latest, windows-latest]
2626
fail-fast: false # Continue running all jobs even if one fails
2727
env:

‎.github/workflows/python-publish.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Python Publish
1+
name: Python Publish (pypi)
22
on:
33
release:
44
types: [created]

‎.github/workflows/python-smoke-tests.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
python-ci:
2222
strategy:
2323
matrix:
24-
python-version: ["3.10", "3.11"] # add 3.12 once gensim supports it. TODO: watch this issue - https://github.com/piskvorky/gensim/issues/3510
24+
python-version: ["3.10"]
2525
os: [ubuntu-latest, windows-latest]
2626
fail-fast: false # Continue running all jobs even if one fails
2727
env:
@@ -65,7 +65,7 @@ jobs:
6565
- '**/*.toml'
6666
- '**/*.ipynb'
6767
- '.github/workflows/python*.yml'
68-
- 'tests/smoke/*'
68+
- 'tests/smoke/**/*'
6969
7070
- name: Set up Python ${{ matrix.python-version }}
7171
uses: actions/setup-python@v5

‎pyproject.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,9 @@ authors = [
1212
"Gaudy Blanco Meneses <gaudyb@microsoft.com>",
1313
"Ha Trinh <trinhha@microsoft.com>",
1414
"Jonathan Larson <jolarso@microsoft.com>",
15+
"Josh Bradley <joshbradley@microsoft.com>",
1516
"Kate Lytvynets <kalytv@microsoft.com>",
17+
"Kenny Zhang <zhangken@microsoft.com>",
1618
"Mónica Carvajal",
1719
"Nathan Evans <naevans@microsoft.com>",
1820
"Rodrigo Racanicci <rracanicci@microsoft.com>",

‎tests/integration/_pipeline/megapipeline.yml‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,14 +39,6 @@ workflows:
3939

4040
- name: create_base_documents
4141

42-
# - name: create_final_community_reports
43-
# config:
44-
# create_community_reports:
45-
# <<: *llm_parallel_config
46-
# strategy:
47-
# type: graph_intelligence
48-
# llm: *llm_config
49-
5042
- name: create_final_communities
5143
- name: create_final_text_units
5244
config:

0 commit comments

Comments
 (0)