Add GPU E2E tests (#1882)

Author: Miguel Varela Ramos

test/e2e/README.md

@@ -32,9 +32,9 @@ Using a new cluster, created for testing only and deleted afterwards:
 pytest test/e2e/tests -k aws --aws-config <cortex_aws_cluster_config.yaml>
 ```
 
-**Note:** For the BatchAPI tests, the `--s3-path` option should be provided with an
-AWS S3 bucket for testing purposes. It is more convenient however to define
-this bucket through an environment variable, see [configuration](#configuration).
+**Note:** For the BatchAPI tests, the `--s3-path` option should be provided with an AWS S3 bucket for testing purposes.
+It is more convenient however to define this bucket through an environment variable, see [configuration](#configuration)
+.
 
 ### GCP
 
@@ -52,19 +52,23 @@ pytest test/e2e/tests -k gcp --gcp-config <cortex_gcp_cluster_config.yaml>
 
 ### All Tests
 
-You can run all tests at once, however the provider specific options should be passed
-accordingly, or the test cases will be skipped.
+You can run all tests at once, however the provider specific options should be passed accordingly, or the test cases
+will be skipped.
 
 e.g.
 
 ```shell
 pytest test/e2e/tests --aws-env <cortex_aws_env> --gcp-env <cortex_gcp_env>
 ```
 
+### Skip GPU Tests
+
+It is possible to skip GPU tests by passing the `--skip-gpus` flag to the pytest command.
+
 ## Configuration
 
-It is possible to configure the behaviour of the tests by defining
-environment variables or a `.env` file at the project directory.
+It is possible to configure the behaviour of the tests by defining environment variables or a `.env` file at the project
+directory.
 
 ```dotenv
 # .env file

test/e2e/tests/aws/test_realtime.py

@@ -19,6 +19,7 @@
 import e2e.tests
 
 TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"]
+TEST_APIS_GPU = ["pytorch/text-generator", "tensorflow/text-generator"]
 
 
 @pytest.mark.usefixtures("client")
@@ -27,3 +28,15 @@ def test_realtime_api(config: Dict, client: cx.Client, api: str):
     e2e.tests.test_realtime_api(
         client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"]
     )
+
+
+@pytest.mark.usefixtures("client")
+@pytest.mark.parametrize("api", TEST_APIS_GPU)
+def test_realtime_api_gpu(config: Dict, client: cx.Client, api: str):
+    skip_gpus = config["global"].get("skip_gpus", False)
+    if skip_gpus:
+        pytest.skip("--skip-gpus flag detected, skipping GPU tests")
+
+    e2e.tests.test_realtime_api(
+        client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"]
+    )

test/e2e/tests/conftest.py

@@ -49,6 +49,11 @@ def pytest_addoption(parser):
         default=None,
         help="set s3 path where batch jobs results will be stored",
     )
+    parser.addoption(
+        "--skip-gpus",
+        action="store_true",
+        help="skip GPU tests",
+    )
 
 
 def pytest_configure(config):
@@ -69,10 +74,11 @@ def pytest_configure(config):
         },
         "global": {
             "realtime_deploy_timeout": int(
-                os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 120)
+                os.environ.get("CORTEX_TEST_REALTIME_DEPLOY_TIMEOUT", 200)
             ),
             "batch_deploy_timeout": int(os.environ.get("CORTEX_TEST_BATCH_DEPLOY_TIMEOUT", 30)),
             "batch_job_timeout": int(os.environ.get("CORTEX_TEST_BATCH_JOB_TIMEOUT", 200)),
+            "skip_gpus": config.getoption("--skip-gpus"),
         },
     }
 

test/e2e/tests/gcp/test_realtime.py

@@ -19,6 +19,7 @@
 import e2e.tests
 
 TEST_APIS = ["pytorch/iris-classifier", "onnx/iris-classifier", "tensorflow/iris-classifier"]
+TEST_APIS_GPU = ["pytorch/text-generator", "tensorflow/text-generator"]
 
 
 @pytest.mark.usefixtures("client")
@@ -27,3 +28,15 @@ def test_realtime_apis(config: Dict, client: cx.Client, api: str):
     e2e.tests.test_realtime_api(
         client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"]
     )
+
+
+@pytest.mark.usefixtures("client")
+@pytest.mark.parametrize("api", TEST_APIS_GPU)
+def test_realtime_api_gpu(config: Dict, client: cx.Client, api: str):
+    skip_gpus = config["global"].get("skip_gpus", False)
+    if skip_gpus:
+        pytest.skip("--skip-gpus flag detected, skipping GPU tests")
+
+    e2e.tests.test_realtime_api(
+        client=client, api=api, timeout=config["global"]["realtime_deploy_timeout"]
+    )