fix: Simplify how we process test dependencies, which are supposed to include all extras. (#4513)

Co-authored-by: Shailav Taneja <none>

Author: ShailavTaneja

requirements/extras/test_requirements.txt

@@ -23,12 +23,7 @@ pyvis==0.2.1
 pandas>=1.3.5,<1.5
 scikit-learn==1.3.0
 cloudpickle==2.2.1
-scipy==1.10.1
-urllib3>=1.26.8,<3.0.0
-docker>=5.0.2,<7.0.0
 PyYAML==6.0
-pyspark==3.3.1
-sagemaker-feature-store-pyspark-3.3
 # TODO find workaround
 xgboost>=1.6.2,<=1.7.6
 pillow>=10.0.1,<=11
@@ -39,4 +34,3 @@ tritonclient[http]<2.37.0
 onnx==1.14.1
 # tf2onnx==1.15.1
 nbformat>=5.9,<6
-accelerate>=0.24.1,<=0.27.0

setup.py

@@ -85,6 +85,8 @@ def read_requirements(filename):
 extras["all"] = [item for group in extras.values() for item in group]
 # Tests specific dependencies (do not need to be included in 'all')
 test_dependencies = read_requirements("requirements/extras/test_requirements.txt")
+# test dependencies are a superset of testing and extra dependencies
+test_dependencies.extend(extras["all"])
 # remove torch and torchvision if python version is not 3.10
 if sys.version_info.minor != 10:
     test_dependencies = [