Skip to content

Commit a926640

Browse files
sentrivanasentrivana
authored
tests: Remove old system for checking for regressions (#4826)
### Description With all eligible test suites now moved to toxgen, we can remove our old way of testing for regressions in new versions of packages. For context, before toxgen we had two groups of test targets for each test suite: pinned and latest. Pinned targets used to be a small selection of hand-picked pinned versions of the package to test. Latest always pulled the newest release, to check for incompatibilities. There was a lot of code to cater to having these two different categories. With the switch to toxgen, all targets are now pinned, and we make sure we pick a representative sample of versions to test against, always including the latest available version to detect regressions. I'm removing the code dealing with the `latest` group entirely, and removing references to anything `pinned` as this is now simply the default. #### Issues <!-- * resolves: #1234 * resolves: LIN-1234 --> #### Reminders - Please add tests to validate your changes, and lint your code using `tox -e linters`. - Add GH Issue ID _&_ Linear ID (if applicable) - PR title should use [conventional commit](https://develop.sentry.dev/engineering-practices/commit-messages/#type) style (`feat:`, `fix:`, `ref:`, `meta:`) - For external contributors: [CONTRIBUTING.md](https://github.com/getsentry/sentry-python/blob/master/CONTRIBUTING.md), [Sentry SDK development docs](https://develop.sentry.dev/sdk/), [Discord community](https://discord.gg/Ww9hbqr)
1 parent 4c8c925 commit a926640

18 files changed

+210
-262
lines changed

‎.github/workflows/test-integrations-ai.yml‎

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ env:
2222
CACHED_BUILD_PATHS: |
2323
${{ github.workspace }}/dist-serverless
2424
jobs:
25-
test-ai-pinned:
26-
name: AI (pinned)
25+
test-ai:
26+
name: AI
2727
timeout-minutes: 30
2828
runs-on: ${{ matrix.os }}
2929
strategy:
@@ -50,42 +50,42 @@ jobs:
5050
- name: Erase coverage
5151
run: |
5252
coverage erase
53-
- name: Test anthropic pinned
53+
- name: Test anthropic
5454
run: |
5555
set -x # print commands that are executed
56-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-anthropic"
57-
- name: Test cohere pinned
56+
./scripts/runtox.sh "py${{ matrix.python-version }}-anthropic"
57+
- name: Test cohere
5858
run: |
5959
set -x # print commands that are executed
60-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-cohere"
61-
- name: Test langchain-base pinned
60+
./scripts/runtox.sh "py${{ matrix.python-version }}-cohere"
61+
- name: Test langchain-base
6262
run: |
6363
set -x # print commands that are executed
64-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain-base"
65-
- name: Test langchain-notiktoken pinned
64+
./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-base"
65+
- name: Test langchain-notiktoken
6666
run: |
6767
set -x # print commands that are executed
68-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langchain-notiktoken"
69-
- name: Test openai-base pinned
68+
./scripts/runtox.sh "py${{ matrix.python-version }}-langchain-notiktoken"
69+
- name: Test openai-base
7070
run: |
7171
set -x # print commands that are executed
72-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai-base"
73-
- name: Test openai-notiktoken pinned
72+
./scripts/runtox.sh "py${{ matrix.python-version }}-openai-base"
73+
- name: Test openai-notiktoken
7474
run: |
7575
set -x # print commands that are executed
76-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai-notiktoken"
77-
- name: Test langgraph pinned
76+
./scripts/runtox.sh "py${{ matrix.python-version }}-openai-notiktoken"
77+
- name: Test langgraph
7878
run: |
7979
set -x # print commands that are executed
80-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-langgraph"
81-
- name: Test openai_agents pinned
80+
./scripts/runtox.sh "py${{ matrix.python-version }}-langgraph"
81+
- name: Test openai_agents
8282
run: |
8383
set -x # print commands that are executed
84-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openai_agents"
85-
- name: Test huggingface_hub pinned
84+
./scripts/runtox.sh "py${{ matrix.python-version }}-openai_agents"
85+
- name: Test huggingface_hub
8686
run: |
8787
set -x # print commands that are executed
88-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-huggingface_hub"
88+
./scripts/runtox.sh "py${{ matrix.python-version }}-huggingface_hub"
8989
- name: Generate coverage XML (Python 3.6)
9090
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
9191
run: |
@@ -114,13 +114,13 @@ jobs:
114114
files: .junitxml
115115
verbose: true
116116
check_required_tests:
117-
name: All pinned AI tests passed
118-
needs: test-ai-pinned
117+
name: All AI tests passed
118+
needs: test-ai
119119
# Always run this, even if a dependent job failed
120120
if: always()
121121
runs-on: ubuntu-22.04
122122
steps:
123123
- name: Check for failures
124-
if: needs.test-ai-pinned.result != 'success'
124+
if: needs.test-ai.result != 'success'
125125
run: |
126126
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

‎.github/workflows/test-integrations-cloud.yml‎

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ env:
2222
CACHED_BUILD_PATHS: |
2323
${{ github.workspace }}/dist-serverless
2424
jobs:
25-
test-cloud-pinned:
26-
name: Cloud (pinned)
25+
test-cloud:
26+
name: Cloud
2727
timeout-minutes: 30
2828
runs-on: ${{ matrix.os }}
2929
strategy:
@@ -54,26 +54,26 @@ jobs:
5454
- name: Erase coverage
5555
run: |
5656
coverage erase
57-
- name: Test aws_lambda pinned
57+
- name: Test aws_lambda
5858
run: |
5959
set -x # print commands that are executed
60-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-aws_lambda"
61-
- name: Test boto3 pinned
60+
./scripts/runtox.sh "py${{ matrix.python-version }}-aws_lambda"
61+
- name: Test boto3
6262
run: |
6363
set -x # print commands that are executed
64-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-boto3"
65-
- name: Test chalice pinned
64+
./scripts/runtox.sh "py${{ matrix.python-version }}-boto3"
65+
- name: Test chalice
6666
run: |
6767
set -x # print commands that are executed
68-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-chalice"
69-
- name: Test cloud_resource_context pinned
68+
./scripts/runtox.sh "py${{ matrix.python-version }}-chalice"
69+
- name: Test cloud_resource_context
7070
run: |
7171
set -x # print commands that are executed
72-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-cloud_resource_context"
73-
- name: Test gcp pinned
72+
./scripts/runtox.sh "py${{ matrix.python-version }}-cloud_resource_context"
73+
- name: Test gcp
7474
run: |
7575
set -x # print commands that are executed
76-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-gcp"
76+
./scripts/runtox.sh "py${{ matrix.python-version }}-gcp"
7777
- name: Generate coverage XML (Python 3.6)
7878
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
7979
run: |
@@ -102,13 +102,13 @@ jobs:
102102
files: .junitxml
103103
verbose: true
104104
check_required_tests:
105-
name: All pinned Cloud tests passed
106-
needs: test-cloud-pinned
105+
name: All Cloud tests passed
106+
needs: test-cloud
107107
# Always run this, even if a dependent job failed
108108
if: always()
109109
runs-on: ubuntu-22.04
110110
steps:
111111
- name: Check for failures
112-
if: needs.test-cloud-pinned.result != 'success'
112+
if: needs.test-cloud.result != 'success'
113113
run: |
114114
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

‎.github/workflows/test-integrations-common.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ env:
2222
CACHED_BUILD_PATHS: |
2323
${{ github.workspace }}/dist-serverless
2424
jobs:
25-
test-common-pinned:
26-
name: Common (pinned)
25+
test-common:
26+
name: Common
2727
timeout-minutes: 30
2828
runs-on: ${{ matrix.os }}
2929
strategy:
@@ -50,10 +50,10 @@ jobs:
5050
- name: Erase coverage
5151
run: |
5252
coverage erase
53-
- name: Test common pinned
53+
- name: Test common
5454
run: |
5555
set -x # print commands that are executed
56-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-common"
56+
./scripts/runtox.sh "py${{ matrix.python-version }}-common"
5757
- name: Generate coverage XML (Python 3.6)
5858
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
5959
run: |
@@ -82,13 +82,13 @@ jobs:
8282
files: .junitxml
8383
verbose: true
8484
check_required_tests:
85-
name: All pinned Common tests passed
86-
needs: test-common-pinned
85+
name: All Common tests passed
86+
needs: test-common
8787
# Always run this, even if a dependent job failed
8888
if: always()
8989
runs-on: ubuntu-22.04
9090
steps:
9191
- name: Check for failures
92-
if: needs.test-common-pinned.result != 'success'
92+
if: needs.test-common.result != 'success'
9393
run: |
9494
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

‎.github/workflows/test-integrations-dbs.yml‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ env:
2222
CACHED_BUILD_PATHS: |
2323
${{ github.workspace }}/dist-serverless
2424
jobs:
25-
test-dbs-pinned:
26-
name: DBs (pinned)
25+
test-dbs:
26+
name: DBs
2727
timeout-minutes: 30
2828
runs-on: ${{ matrix.os }}
2929
strategy:
@@ -70,30 +70,30 @@ jobs:
7070
- name: Erase coverage
7171
run: |
7272
coverage erase
73-
- name: Test asyncpg pinned
73+
- name: Test asyncpg
7474
run: |
7575
set -x # print commands that are executed
76-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-asyncpg"
77-
- name: Test clickhouse_driver pinned
76+
./scripts/runtox.sh "py${{ matrix.python-version }}-asyncpg"
77+
- name: Test clickhouse_driver
7878
run: |
7979
set -x # print commands that are executed
80-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-clickhouse_driver"
81-
- name: Test pymongo pinned
80+
./scripts/runtox.sh "py${{ matrix.python-version }}-clickhouse_driver"
81+
- name: Test pymongo
8282
run: |
8383
set -x # print commands that are executed
84-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-pymongo"
85-
- name: Test redis pinned
84+
./scripts/runtox.sh "py${{ matrix.python-version }}-pymongo"
85+
- name: Test redis
8686
run: |
8787
set -x # print commands that are executed
88-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-redis"
89-
- name: Test redis_py_cluster_legacy pinned
88+
./scripts/runtox.sh "py${{ matrix.python-version }}-redis"
89+
- name: Test redis_py_cluster_legacy
9090
run: |
9191
set -x # print commands that are executed
92-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-redis_py_cluster_legacy"
93-
- name: Test sqlalchemy pinned
92+
./scripts/runtox.sh "py${{ matrix.python-version }}-redis_py_cluster_legacy"
93+
- name: Test sqlalchemy
9494
run: |
9595
set -x # print commands that are executed
96-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-sqlalchemy"
96+
./scripts/runtox.sh "py${{ matrix.python-version }}-sqlalchemy"
9797
- name: Generate coverage XML (Python 3.6)
9898
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
9999
run: |
@@ -122,13 +122,13 @@ jobs:
122122
files: .junitxml
123123
verbose: true
124124
check_required_tests:
125-
name: All pinned DBs tests passed
126-
needs: test-dbs-pinned
125+
name: All DBs tests passed
126+
needs: test-dbs
127127
# Always run this, even if a dependent job failed
128128
if: always()
129129
runs-on: ubuntu-22.04
130130
steps:
131131
- name: Check for failures
132-
if: needs.test-dbs-pinned.result != 'success'
132+
if: needs.test-dbs.result != 'success'
133133
run: |
134134
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

‎.github/workflows/test-integrations-flags.yml‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ env:
2222
CACHED_BUILD_PATHS: |
2323
${{ github.workspace }}/dist-serverless
2424
jobs:
25-
test-flags-pinned:
26-
name: Flags (pinned)
25+
test-flags:
26+
name: Flags
2727
timeout-minutes: 30
2828
runs-on: ${{ matrix.os }}
2929
strategy:
@@ -50,22 +50,22 @@ jobs:
5050
- name: Erase coverage
5151
run: |
5252
coverage erase
53-
- name: Test launchdarkly pinned
53+
- name: Test launchdarkly
5454
run: |
5555
set -x # print commands that are executed
56-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-launchdarkly"
57-
- name: Test openfeature pinned
56+
./scripts/runtox.sh "py${{ matrix.python-version }}-launchdarkly"
57+
- name: Test openfeature
5858
run: |
5959
set -x # print commands that are executed
60-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-openfeature"
61-
- name: Test statsig pinned
60+
./scripts/runtox.sh "py${{ matrix.python-version }}-openfeature"
61+
- name: Test statsig
6262
run: |
6363
set -x # print commands that are executed
64-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-statsig"
65-
- name: Test unleash pinned
64+
./scripts/runtox.sh "py${{ matrix.python-version }}-statsig"
65+
- name: Test unleash
6666
run: |
6767
set -x # print commands that are executed
68-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-unleash"
68+
./scripts/runtox.sh "py${{ matrix.python-version }}-unleash"
6969
- name: Generate coverage XML (Python 3.6)
7070
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
7171
run: |
@@ -94,13 +94,13 @@ jobs:
9494
files: .junitxml
9595
verbose: true
9696
check_required_tests:
97-
name: All pinned Flags tests passed
98-
needs: test-flags-pinned
97+
name: All Flags tests passed
98+
needs: test-flags
9999
# Always run this, even if a dependent job failed
100100
if: always()
101101
runs-on: ubuntu-22.04
102102
steps:
103103
- name: Check for failures
104-
if: needs.test-flags-pinned.result != 'success'
104+
if: needs.test-flags.result != 'success'
105105
run: |
106106
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

‎.github/workflows/test-integrations-gevent.yml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,8 @@ env:
2222
CACHED_BUILD_PATHS: |
2323
${{ github.workspace }}/dist-serverless
2424
jobs:
25-
test-gevent-pinned:
26-
name: Gevent (pinned)
25+
test-gevent:
26+
name: Gevent
2727
timeout-minutes: 30
2828
runs-on: ${{ matrix.os }}
2929
strategy:
@@ -50,10 +50,10 @@ jobs:
5050
- name: Erase coverage
5151
run: |
5252
coverage erase
53-
- name: Test gevent pinned
53+
- name: Test gevent
5454
run: |
5555
set -x # print commands that are executed
56-
./scripts/runtox.sh --exclude-latest "py${{ matrix.python-version }}-gevent"
56+
./scripts/runtox.sh "py${{ matrix.python-version }}-gevent"
5757
- name: Generate coverage XML (Python 3.6)
5858
if: ${{ !cancelled() && matrix.python-version == '3.6' }}
5959
run: |
@@ -82,13 +82,13 @@ jobs:
8282
files: .junitxml
8383
verbose: true
8484
check_required_tests:
85-
name: All pinned Gevent tests passed
86-
needs: test-gevent-pinned
85+
name: All Gevent tests passed
86+
needs: test-gevent
8787
# Always run this, even if a dependent job failed
8888
if: always()
8989
runs-on: ubuntu-22.04
9090
steps:
9191
- name: Check for failures
92-
if: needs.test-gevent-pinned.result != 'success'
92+
if: needs.test-gevent.result != 'success'
9393
run: |
9494
echo "One of the dependent jobs has failed. You may need to re-run it." && exit 1

0 commit comments

Comments
 (0)