Kcz/master/support for video in benchmark

tests/python_tests/samples/conftest.py

@@ -26,7 +26,7 @@
 # - "name": the model's name or path
 # - "convert_args": a list of arguments for the conversion command
 MODELS: Dict[str, Dict[str, Any]] = {
-    "TinyLlama-1.1B-Chat-v1.0": { 
+    "TinyLlama-1.1B-Chat-v1.0": {
         "name": "TinyLlama/TinyLlama-1.1B-Chat-v1.0",
         "convert_args": ['--weight-format', 'fp16']
     },
@@ -46,7 +46,7 @@
     "SmolLM2-360M": {
         "name": "HuggingFaceTB/SmolLM2-360M",
         "convert_args": ['--trust-remote-code']
-    },  
+    },
     "WhisperTiny": {
         "name": "openai/whisper-tiny",
         "convert_args": ['--trust-remote-code', '--weight-format', 'fp16']
@@ -84,11 +84,11 @@
     "LCM_Dreamshaper_v7-int8-ov": {
         "name": "OpenVINO/LCM_Dreamshaper_v7-int8-ov",
         "convert_args": []
-    },   
+    },
     "llava-1.5-7b-hf": {
         "name": "llava-hf/llava-1.5-7b-hf",
         "convert_args": ['--trust-remote-code', '--weight-format', 'fp16']
-    },    
+    },
     "llava-v1.6-mistral-7b-hf": {
         "name": "llava-hf/llava-v1.6-mistral-7b-hf",
         "convert_args": ['--trust-remote-code', '--weight-format', 'fp16']
@@ -129,6 +129,10 @@
         "name": "katuni4ka/tiny-random-llava",
         "convert_args": ["--trust-remote-code", "--task", "image-text-to-text"]
     },
+    "tiny-random-qwen2vl": {
+        "name": "katuni4ka/tiny-random-qwen2vl",
+        "convert_args": ["--trust-remote-code", "--task", "image-text-to-text"]
+    },
     "bge-small-en-v1.5": {
         "name": "BAAI/bge-small-en-v1.5",
         "convert_args": ["--trust-remote-code"]
@@ -148,6 +152,10 @@
     "tiny-random-SpeechT5ForTextToSpeech": {
         "name": "hf-internal-testing/tiny-random-SpeechT5ForTextToSpeech",
         "convert_args": ["--model-kwargs",  json.dumps({"vocoder": "fxmarty/speecht5-hifigan-tiny"})]
+    },
+    "tiny-random-llava-next-video": {
+        "name": "katuni4ka/tiny-random-llava-next-video",
+        "convert_args": ["--trust-remote-code", "--task", "image-text-to-text"]
     }
 }
 
@@ -164,7 +172,8 @@
     "cat.png": "https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/diffusers/cat.png",
     "cat": "https://github.com/openvinotoolkit/openvino_notebooks/assets/29454499/d5fbbd1a-d484-415c-88cb-9986625b7b11",
     "3283_1447_000.tar.gz": "https://huggingface.co/datasets/facebook/multilingual_librispeech/resolve/main/data/mls_polish/train/audio/3283_1447_000.tar.gz",
-    "cmu_us_awb_arctic-wav-arctic_a0001.bin": "https://huggingface.co/datasets/Xenova/cmu-arctic-xvectors-extracted/resolve/main/cmu_us_awb_arctic-wav-arctic_a0001.bin"
+    "cmu_us_awb_arctic-wav-arctic_a0001.bin": "https://huggingface.co/datasets/Xenova/cmu-arctic-xvectors-extracted/resolve/main/cmu_us_awb_arctic-wav-arctic_a0001.bin",
+    "video0.mp4": "https://storage.openvinotoolkit.org/repositories/openvino_notebooks/data/data/video/Coco%20Walking%20in%20Berkeley.mp4"
 }
 
 SAMPLES_PY_DIR = Path(
@@ -182,23 +191,24 @@
     )
 )
 
+
 @pytest.fixture(scope="session", autouse=True)
 def setup_and_teardown(request, tmp_path_factory):
     """Fixture to set up and tear down the temporary directories."""
-    
-    ov_cache = get_ov_cache_dir(tmp_path_factory.mktemp("ov_cache"))  
+
+    ov_cache = get_ov_cache_dir(tmp_path_factory.mktemp("ov_cache"))
     downloaded_models_dir = get_ov_cache_downloaded_models_dir()
     converted_models_dir = get_ov_cache_converted_models_dir()
     test_data = ov_cache / "test_data"
-    
+
     logger.info(f"Creating directories: {downloaded_models_dir}, {converted_models_dir}, and {test_data}")
     test_data.mkdir(parents=True, exist_ok=True)
-    
+
     request.config.cache.set("OV_CACHE", str(ov_cache))
     request.config.cache.set("TEST_DATA", str(test_data))
-    
+
     yield
-    
+
     if os.environ.get("CLEANUP_CACHE", "false").lower() != "false":
         if os.path.exists(ov_cache):
             logger.info(f"Removing temporary directory: {ov_cache}")
@@ -213,9 +223,9 @@ def download_gguf_model(model: Dict[str, Any], model_path: str) -> None:
     model_name = model["name"]
     model_gguf_filename = model["gguf_filename"]
     dest_dir = Path(model_path)
-    
+
     manager = AtomicDownloadManager(dest_dir)
-    
+
     def download_to_temp(temp_path: Path) -> None:
         command = ["huggingface-cli", "download", model_name, model_gguf_filename, "--local-dir", str(temp_path)]
         logger.info(f"Downloading command: {' '.join(command)}")
@@ -325,26 +335,27 @@ def download_to_temp(temp_path: Path) -> None:
         command = ["huggingface-cli", "download", model_name, "--local-dir", str(temp_path)]
         logger.info(f"Downloading command: {' '.join(command)}")
         retry_request(lambda: subprocess.run(command, check=True, capture_output=True, text=True, env=sub_env))
-    
+
     manager.execute(download_to_temp)
-            
+
     yield str(model_path)
-    
+
     if os.environ.get("CLEANUP_CACHE", "false").lower() == "true":
         if model_cache.exists():
             logger.info(f"Removing downloaded model: {model_cache}")
             shutil.rmtree(model_cache)
 
+
 @pytest.fixture(scope="session")
 def download_test_content(request):
     """Download the test content from the given URL and return the file path or extracted folder."""
-    
+
     test_data = request.config.cache.get("TEST_DATA", None)
-    
+
     file_name = request.param
     file_url = TEST_FILES[file_name]
     file_path = os.path.join(test_data, file_name)
-    
+
     if not os.path.exists(file_path):
         logger.info(f"Downloading test content from {file_url} to {file_path}...")
         os.makedirs(os.path.dirname(file_path), exist_ok=True)
@@ -384,9 +395,9 @@ def download_test_content(request):
 @pytest.fixture(scope="session")
 def generate_test_content(request):
     """Generate an image of lines and return the file path."""
-    
+
     test_data = request.config.cache.get("TEST_DATA", None)
-    
+
     file_name = request.param
     file_path = os.path.join(test_data, file_name)
     if not os.path.exists(file_path):
@@ -412,24 +423,24 @@ def generate_test_content(request):
 @pytest.fixture(scope="session")
 def generate_image_generation_jsonl(request):
     """Generate a JSONL file for image generation prompts."""
-    
+
     test_data = request.config.cache.get("TEST_DATA", None)
     file_name, json_entries = request.param
     file_path = os.path.join(test_data, file_name)
-    
+
     if not os.path.exists(file_path):
         os.makedirs(os.path.dirname(file_path), exist_ok=True)
-        
+
         with open(file_path, "w", encoding="utf-8") as f:
             for entry in json_entries:
                 f.write(json.dumps(entry) + "\n")
-        
+
         logger.info(f"Generated image generation JSONL file at {file_path}")
     else:
         logger.info(f"Image generation JSONL file already exists at {file_path}")
-    
+
     yield file_path
-    
+
     # Cleanup the JSONL file after tests
     if os.environ.get("CLEANUP_CACHE", "false").lower() == "true":
         if os.path.exists(file_path):