Base LLM pipeline testing refactoring and optimization (#2779)

## Description
Ticket: 
Part of [CVS-173285](https://jira.devtools.intel.com/browse/CVS-173285)

Author: sgonorov

tests/python_tests/README.md

@@ -39,7 +39,7 @@ Argument `--model_ids` can be used to run tests selectively only for specific mo
 python -m pytest tests/python_tests/ -k "test_multibatch" --model_ids "TinyLlama/TinyLlama-1.1B-Chat-v1.0 Qwen/Qwen2-0.5B-Instruct"
 ```
 
-List of currently supported models can be found in tests/python_tests/models.py:get_models_list
+List of currently supported models can be found in tests/python_tests/data/models.py:get_models_list
 
 ## Test Samples
 To test samples, set the `SAMPLES_PY_DIR` and `SAMPLES_CPP_DIR` environment variables to the directories containing your Python samples and built C++ samples respectively. The `SAMPLES_CPP_DIR` should point to the folder with built C++ samples, which can be installed using `cmake --component samples_bin`. For example:

tests/python_tests/conftest.py

@@ -1,8 +1,13 @@
 import os
+import gc
 import pytest
 import shutil
 import logging
-from utils.constants import get_ov_cache_models_dir
+from utils.constants import (
+    get_ov_cache_dir,
+    get_ov_cache_downloaded_models_dir,
+    get_ov_cache_converted_models_dir,
+)
 
 # Configure logging
 logging.basicConfig(level=logging.INFO)
@@ -13,20 +18,23 @@
 def setup_and_teardown():
     """Fixture to set up and tear down the temporary directories."""
 
-    ov_cache_models_dir = get_ov_cache_models_dir()
+    ov_cache_dir = get_ov_cache_dir()
+    ov_cache_downloaded_dir = get_ov_cache_downloaded_models_dir()
+    ov_cache_converted_dir = get_ov_cache_converted_models_dir()
 
-    logger.info(f"Creating directory: {ov_cache_models_dir}")
-    os.makedirs(ov_cache_models_dir, exist_ok=True)
+    logger.info(f"Creating directories: {ov_cache_downloaded_dir}, {ov_cache_converted_dir}")
+    ov_cache_downloaded_dir.mkdir(exist_ok=True, parents=True)
+    ov_cache_converted_dir.mkdir(exist_ok=True, parents=True)
 
     yield
 
     if os.environ.get("CLEANUP_CACHE", "false").lower() != "false":
-        if os.path.exists(ov_cache_models_dir):
-            logger.info(f"Removing temporary directory: {ov_cache_models_dir}")
-            shutil.rmtree(ov_cache_models_dir)
+        if ov_cache_dir.exists():
+            logger.info(f"Removing temporary directory: {ov_cache_dir}")
+            shutil.rmtree(ov_cache_dir)
         else:
             logger.info(
-                f"Skipped temporary directory cleanup because it doesn't exist: {ov_cache_models_dir}"
+                f"Skipped temporary directory cleanup because it doesn't exist: {ov_cache_dir}"
             )
 
 
@@ -54,3 +62,14 @@ def pytest_addoption(parser):
 
 def pytest_configure(config: pytest.Config):
     pytest.selected_model_ids = config.getoption("--model_ids", default=None)
+
+
+@pytest.fixture(scope="module", autouse=True)
+def run_gc_after_test():
+    """
+    Fixture to run garbage collection after each test module.
+    This is a workaround to minimize memory consumption during tests 
+    and allow the use of less powerful CI runners.
+    """
+    yield
+    gc.collect()

tests/python_tests/data/models.py

@@ -1,53 +1,48 @@
 # Copyright (C) 2024 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
-from pathlib import Path
 import pytest
-from os.path import sep
 
 
-def get_models_list():
-    model_ids = [
+def get_models_list() -> tuple[str, ...]:
+    model_ids: list[str] = [
         "katuni4ka/tiny-random-phi3",
     ]
     if pytest.selected_model_ids:
         model_ids = [model_id for model_id in model_ids if model_id in pytest.selected_model_ids.split(' ')]
 
-    return model_ids
+    return tuple(model_ids)
 
 
-def get_chat_models_list():
-    return [
-        "Qwen/Qwen2-0.5B-Instruct",
-    ]
+CHAT_MODELS_LIST = (
+    "Qwen/Qwen2-0.5B-Instruct",
+)
 
 
-def get_gguf_model_list():
-    return [
+GGUF_MODEL_LIST = (
+    {
+        "hf_model_id": "HuggingFaceTB/SmolLM2-135M",
+        "gguf_model_id": "prithivMLmods/SmolLM2-135M-GGUF",
+        "gguf_filename": "SmolLM2-135M.F16.gguf",
+        "dynamic_quantization_group_size": None,
+    },
+    {
+        "gguf_model_id": "Qwen/Qwen2.5-0.5B-Instruct-GGUF",
+        "gguf_filename": "qwen2.5-0.5b-instruct-q4_0.gguf",
+        "dynamic_quantization_group_size": None,
+    },
+    pytest.param(
         {
             "hf_model_id": "HuggingFaceTB/SmolLM2-135M",
-            "gguf_model_id": "prithivMLmods/SmolLM2-135M-GGUF",
-            "gguf_filename": "SmolLM2-135M.F16.gguf",
-            "dynamic_quantization_group_size": None,
-        },
-        {
-            "gguf_model_id": "Qwen/Qwen2.5-0.5B-Instruct-GGUF",
-            "gguf_filename": "qwen2.5-0.5b-instruct-q4_0.gguf",
+            "gguf_model_id": "QuantFactory/SmolLM2-135M-GGUF",
+            "gguf_filename": "SmolLM2-135M.Q4_1.gguf",
             "dynamic_quantization_group_size": None,
         },
-        pytest.param(
-            {
-                "hf_model_id": "HuggingFaceTB/SmolLM2-135M",
-                "gguf_model_id": "QuantFactory/SmolLM2-135M-GGUF",
-                "gguf_filename": "SmolLM2-135M.Q4_1.gguf",
-                "dynamic_quantization_group_size": None,
-            },
-            marks=pytest.mark.xfail(reason="Prediction mismatch. Ticket 172345", raises=AssertionError),
-        ),
-        {
-            "gguf_model_id": "sammysun0711/tiny-random-deepseek-distill-qwen-gguf",
-            "gguf_filename": "tiny-random-deepseek-distill-qwen_q8_0.gguf",
-            # Dummy gguf model accuracy is sensitive for dynamic quantization w/ small group size 32 (default), set group size as 64 explicitly instead
-            "dynamic_quantization_group_size": "64",
-        },
-    ]
+        marks=pytest.mark.xfail(reason="Prediction mismatch. Ticket 172345", raises=AssertionError),
+    ),
+    {
+        "gguf_model_id": "sammysun0711/tiny-random-deepseek-distill-qwen-gguf",
+        "gguf_filename": "tiny-random-deepseek-distill-qwen_q8_0.gguf",
+        "dynamic_quantization_group_size": "64",
+    },
+)

tests/python_tests/data/test_dataset.py

@@ -2,7 +2,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from openvino_genai import GenerationConfig
-from utils.generation_config import get_greedy, get_beam_search, get_multinomial_temperature
+from utils.generation_config import get_greedy, get_beam_search
 
 def get_test_dataset() -> tuple[list[str], list[GenerationConfig]]:
     prompts = [
@@ -11,10 +11,12 @@ def get_test_dataset() -> tuple[list[str], list[GenerationConfig]]:
         "What is your name?",
         "Tell me something about Canada"
     ]
+
     generation_configs = [
         get_greedy(),
         get_beam_search(),
         get_greedy(),
         get_beam_search(),
     ]
+
     return (prompts, generation_configs)