fix(ci): calculate parallel jobs based on infrastructure needs (#1475)

Author: heitorlessa

.github/workflows/run-e2e-tests.yml

@@ -21,8 +21,8 @@ jobs:
     strategy:
       matrix:
         # Maintenance: disabled until we discover concurrency lock issue with multiple versions and tmp
-        # version: ["3.7", "3.8", "3.9"]
-        version: ["3.7"]
+        version: ["3.7", "3.8", "3.9"]
+        # version: ["3.7"]
     steps:
       - name: "Checkout"
         uses: actions/checkout@v3

Makefile

@@ -35,7 +35,7 @@ unit-test:
 	poetry run pytest tests/unit
 
 e2e-test:
-	poetry run pytest -rP -n auto --dist loadfile -o log_cli=true tests/e2e
+	python parallel_run_e2e.py
 
 coverage-html:
 	poetry run pytest -m "not perf" --ignore tests/e2e --cov=aws_lambda_powertools --cov-report=html

parallel_run_e2e.py

@@ -0,0 +1,16 @@
+""" Calculate how many parallel workers are needed to complete E2E infrastructure jobs across available CPU Cores """
+import subprocess
+from pathlib import Path
+
+
+def main():
+    features = Path("tests/e2e").rglob("infrastructure.py")
+    workers = len(list(features)) - 1
+
+    command = f"poetry run pytest -n {workers} --dist loadfile -o log_cli=true tests/e2e"
+    print(f"Running E2E tests with: {command}")
+    subprocess.run(command.split(), shell=False)
+
+
+if __name__ == "__main__":
+    main()

tox.ini

@@ -0,0 +1,15 @@
+[tox]
+envlist = py37,py38,py39
+
+[testenv]
+deps =
+    filelock
+    pytest-xdist
+    pydantic
+    email-validator
+
+commands = python parallel_run_e2e.py
+
+; If you ever encounter another parallel lock across interpreters
+; pip install tox tox-poetry
+; tox -p --parallel-live