kcz/support-for-video-in-benchmark

Author: krzyczar

tools/llm_bench/benchmark.py

@@ -229,6 +229,8 @@ def get_argprser():
                         help="Path to .bin or .pt file with speaker embeddings for text to speech scenarios")
     parser.add_argument("--vocoder_path", type=str, default=None,
                         help="Path to vocoder  for text to speech scenarios")
+    parser.add_argument("-vf", "--video_frames", type=int, default=None,
+                        help="controler of video frames to process")
     return parser.parse_args()
 
 
@@ -315,7 +317,8 @@ def main():
                 args.num_iters, memory_data_collector)
         else:
             iter_data_list, pretrain_time, iter_timestamp = CASE_TO_BENCH[model_args['use_case'].task](
-                model_path, framework, args.device, model_args, args.num_iters, memory_data_collector)
+                model_path, framework, args.device, model_args, args.num_iters,
+                memory_data_collector, args.video_frames)
         if args.report is not None or args.report_json is not None:
             model_precision = ''
             if framework == 'ov':

tools/llm_bench/llm_bench_utils/parse_json_data.py

@@ -2,78 +2,67 @@
 # Copyright (C) 2023-2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+def create_base_prompt(json_data, key='prompt'):
+    prompt_data = {}
+    if key in json_data:
+        if json_data[key] != "":
+            prompt_data[key] = json_data[key]
+        else:
+            raise RuntimeError(f"== {key} should not be empty string ==")
+    else:
+        raise RuntimeError(f"== key word '{key}' does not exist ==")
+    return prompt_data
+
 
 def parse_text_json_data(json_data_list):
     text_param_list = []
     for json_data in json_data_list:
-        if 'prompt' in json_data:
-            if json_data['prompt'] != '':
-                text_param_list.append(json_data['prompt'])
-            else:
-                raise RuntimeError('== prompt should not be empty string ==')
-        else:
-            raise RuntimeError('== key word "prompt" does not exist ==')
+        prompt_data = create_base_prompt(json_data)
+        text_param_list.append(prompt_data["prompt"])
     return text_param_list
 
 
 def parse_vlm_json_data(json_data_list):
     text_param_list = []
     for json_data in json_data_list:
-        prompt_data = {}
-        if 'prompt' in json_data:
-            if json_data['prompt'] != '':
-                prompt_data["prompt"] = json_data['prompt']
-            else:
-                raise RuntimeError('== prompt should not be empty string ==')
-        else:
-            raise RuntimeError('== key word "prompt" does not exist ==')
+        prompt_data = create_base_prompt(json_data)
+        if ("media" in json_data) and ("video" in json_data):
+            raise ValueError("only one key is avaialble from media & video")
         if "media" in json_data:
             prompt_data["media"] = json_data["media"]
+        if "video" in json_data:
+            prompt_data["video"] = json_data["video"]
         text_param_list.append(prompt_data)
     return text_param_list
 
 
 def parse_image_json_data(json_data_list):
     image_param_list = []
-    for data in json_data_list:
-        image_param = {}
-        if 'prompt' in data:
-            if data['prompt'] != '':
-                image_param['prompt'] = data['prompt']
-            else:
-                raise RuntimeError('== prompt should not be empty string ==')
-        else:
-            raise RuntimeError('== key word "prompt" does not exist in prompt file ==')
-        if 'width' in data:
-            image_param['width'] = int(data['width'])
-        if 'height' in data:
-            image_param['height'] = int(data['height'])
-        if 'steps' in data:
-            image_param['steps'] = int(data['steps'])
-        if 'guidance_scale' in data:
-            image_param['guidance_scale'] = float(data['guidance_scale'])
-        if 'media' in data:
-            image_param['media'] = data['media']
-        if 'mask_image' in data:
-            image_param['mask_image'] = data['mask_image']
+    for json_data in json_data_list:
+        image_param = create_base_prompt(json_data)
+        if 'width' in json_data:
+            image_param['width'] = int(json_data['width'])
+        if 'height' in json_data:
+            image_param['height'] = int(json_data['height'])
+        if 'steps' in json_data:
+            image_param['steps'] = int(json_data['steps'])
+        if 'guidance_scale' in json_data:
+            image_param['guidance_scale'] = float(json_data['guidance_scale'])
+        if 'media' in json_data:
+            image_param['media'] = json_data['media']
+        if 'mask_image' in json_data:
+            image_param['mask_image'] = json_data['mask_image']
         image_param_list.append(image_param)
     return image_param_list
 
 
 def parse_speech_json_data(json_data_list):
     speech_param_list = []
     for json_data in json_data_list:
-        speech_param = {}
-        if 'media' in json_data:
-            if json_data['media'] != '':
-                speech_param['media'] = json_data['media']
-            else:
-                raise RuntimeError('== media path should not be empty string ==')
-        else:
-            raise RuntimeError('== key word "media" does not exist ==')
-        if 'language' in json_data:
-            speech_param['language'] = json_data['language']
-        if 'timestamp' in json_data:
-            speech_param['timestamp'] = json_data['timestamp']
+        speech_param = create_base_prompt(json_data, "media")
+        if "language" in json_data:
+            speech_param["language"] = json_data["language"]
+        if "timestamp" in json_data:
+            speech_param["timestamp"] = json_data["timestamp"]
         speech_param_list.append(speech_param)
     return speech_param_list

tools/llm_bench/llm_bench_utils/prompt_utils.py

@@ -2,6 +2,12 @@
 # Copyright (C) 2023-2025 Intel Corporation
 # SPDX-License-Identifier: Apache-2.0
 
+
+import os
+import cv2
+import numpy as np
+from PIL import Image
+import logging as log
 from .model_utils import get_param_from_file
 from .parse_json_data import parse_text_json_data
 
@@ -17,3 +23,64 @@ def get_text_prompt(args):
     else:
         text_list.append(output_data_list[0])
     return text_list
+
+
+def print_video_frames_number_and_convert_to_tensor(func):
+    def inner(video_path, decym_frames):
+        log.info(f"Input video file: {video_path}")
+        if decym_frames is not None:
+            log.info(f"Requested to reduce into {decym_frames} frames")
+        out_frames = func(video_path, decym_frames)
+        log.info(f"Final frames number: {len(out_frames)}")
+        return np.array(out_frames)
+    return inner
+
+
+@print_video_frames_number_and_convert_to_tensor
+def make_video_tensor(video_path, decym_frames=None):
+    supported_files = set([".mp4"])
+
+    assert os.path.exists(video_path), f"no input video file: {video_path}"
+    assert video_path.suffix.lower() in supported_files, "no supported video file"
+    cap = cv2.VideoCapture(video_path)
+
+    output_frames = []
+    while True:
+        ret, frame = cap.read()
+        if not ret:
+            break
+        frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        pil_image = Image.fromarray(frame_rgb)
+
+        shape = np.array(pil_image).shape
+        dtype = np.array(pil_image).dtype
+        log.info(f"Video shape: {shape}")
+        log.info(f"Video dtype: {dtype}")
+        new_frame = np.zeros(shape, dtype)
+
+        width, height = pil_image.size
+        log.info(f"Video size: {width}x{height}")
+        for x in range(0, width):
+            for y in range(0, height):
+                new_frame[y, x] = frame_rgb[y, x]
+        output_frames.append(np.array(pil_image))
+
+    if decym_frames is None:
+        return output_frames
+    if int(decym_frames) == 0:
+        return output_frames
+
+    # decimation procedure
+    # decim_fames is required frame number if positive
+    # or decimation factor if negative
+
+    decym_frames = int(decym_frames)
+    if decym_frames > 0:
+        if len(output_frames) <= decym_frames:
+            return output_frames
+        decym_factor = int(len(output_frames) / decym_frames)
+    else:
+        decym_factor = -decym_frames
+    if decym_factor >= 2:
+        return output_frames[::decym_factor]
+    return output_frames

tools/llm_bench/requirements.txt

@@ -8,7 +8,7 @@ pillow
 torch
 transformers[sentencepiece]>=4.40.0
 diffusers>=0.22.0
-#optimum is in dependency list of optimum-intel 
+#optimum is in dependency list of optimum-intel
 optimum-intel[nncf]>=1.25.0
 packaging
 psutil
@@ -21,3 +21,4 @@ scipy
 gguf_parser
 gguf>=0.10
 num2words
+opencv-python

tools/llm_bench/task/visual_language_generation.py

@@ -17,27 +17,32 @@
 import llm_bench_utils.output_file
 import llm_bench_utils.gen_output_data as gen_output_data
 import llm_bench_utils.parse_json_data as parse_json_data
+import llm_bench_utils.prompt_utils as pu
 from pathlib import Path
 
-
 FW_UTILS = {'pt': llm_bench_utils.pt_utils, 'ov': llm_bench_utils.ov_utils}
 
 DEFAULT_OUTPUT_TOKEN_SIZE = 512
 
 
 def run_visual_language_generation_optimum(
-    inputs, num, model, processor, args, iter_data_list, md5_list, prompt_index, bench_hook, model_precision, proc_id, mem_consumption
-):
+        inputs, num, model, processor, args, iter_data_list, md5_list, prompt_index,
+        bench_hook, model_precision, proc_id, mem_consumption, required_frames=None):
     from optimum.intel.utils.import_utils import is_transformers_version
     set_seed(args['seed'])
     if args['batch_size'] != 1:
         log.warning("Only batch size 1 available for benchmarking")
         args["batch_size"] = 1
     images = []
     prompts = []
+    videos = []
     inputs = [inputs] if not isinstance(inputs, (list, tuple)) else inputs
     for input_data in inputs:
-        if input_data.get("media", None):
+        if input_data.get("video", None):
+            entry = Path(input_data["video"])
+            video_tensor = pu.make_video_tensor(entry, required_frames)
+            videos.append(video_tensor)
+        elif input_data.get("media", None):
             entry = Path(input_data["media"])
             if entry.is_dir():
                 for file in sorted(entry.iterdir()):
@@ -52,6 +57,8 @@ def run_visual_language_generation_optimum(
             llm_bench_utils.output_file.output_input_text(in_text, args, model_precision, prompt_index, bs_index, proc_id)
     tok_encode_start = time.perf_counter()
     input_data = model.preprocess_inputs(text=prompts[0], image=images[0] if images else None, **processor)
+    if videos:
+        input_data["videos"] = videos
     tok_encode_end = time.perf_counter()
     tok_encode_time = (tok_encode_end - tok_encode_start) * 1000
     # Remove `token_type_ids` from inputs
@@ -189,16 +196,21 @@ def load_image_genai(image_path):
 
 
 def run_visual_language_generation_genai(
-    inputs, num, model, processor, args, iter_data_list, md5_list, prompt_index, streamer, model_precision, proc_id, mem_consumption
-):
+        inputs, num, model, processor, args, iter_data_list, md5_list, prompt_index,
+        streamer, model_precision, proc_id, mem_consumption, required_frames=None):
     if args['batch_size'] != 1:
         log.warning("Only batch size 1 available for benchmarking")
         args["batch_size"] = 1
     images = []
     prompts = []
+    videos = []
     inputs = [inputs] if not isinstance(inputs, (list, tuple)) else inputs
     for input_data in inputs:
-        if input_data.get("media", None):
+        if input_data.get("video", None):
+            entry = Path(input_data["video"])
+            video_tensor = pu.make_video_tensor(entry, required_frames)
+            videos.append(video_tensor)
+        elif input_data.get("media", None):
             entry = Path(input_data["media"])
             if entry.is_dir():
                 for file in sorted(entry.iterdir()):
@@ -222,8 +234,10 @@ def run_visual_language_generation_genai(
     gen_config.do_sample = False
     gen_config.ignore_eos = True
     kwargs = {}
-    if len(images) >= 1:
+    if images:
         kwargs["images"] = images
+    if videos:
+        kwargs["videos"] = videos
     prefix = '[warm-up]' if num == 0 else '[{}]'.format(num)
     log.info(f'{prefix}[P{prompt_index}] Input image nums:{len(images)}')
     start = time.perf_counter()
@@ -304,8 +318,11 @@ def run_visual_language_generation_genai(
         metrics_print.print_generated(num, warm_up=(num == 0), generated=generated_text[0], prompt_idx=prompt_index)
 
 
-def run_visual_language_generation_benchmark(model_path, framework, device, args, num_iters, mem_consumption):
-    model, processor, pretrain_time, bench_hook, use_genai = FW_UTILS[framework].create_image_text_gen_model(model_path, device, mem_consumption, **args)
+def run_visual_language_generation_benchmark(
+        model_path, framework, device, args, num_iters,
+        mem_consumption, required_frames=None):
+    outs = FW_UTILS[framework].create_image_text_gen_model(model_path, device, mem_consumption, **args)
+    model, processor, pretrain_time, bench_hook, use_genai = outs
     model_precision = model_utils.get_model_precision(model_path.parts)
     iter_data_list = []
     md5_list = {num : {} for num in range(num_iters + 1)}
@@ -325,10 +342,10 @@ def run_visual_language_generation_benchmark(model_path, framework, device, args
     log.info(f"Numbeams: {args['num_beams']}, benchmarking iter nums(exclude warm-up): {num_iters}, "
              f'prompt nums: {len(image_text_list)}, prompt idx: {prompt_idx_list}')
 
-    if not use_genai:
-        gen_fn = run_visual_language_generation_optimum
-    else:
+    if use_genai:
         gen_fn = run_visual_language_generation_genai
+    else:
+        gen_fn = run_visual_language_generation_optimum
 
     proc_id = os.getpid()
     iter_timestamp = model_utils.init_timestamp(num_iters, image_text_list, prompt_idx_list)
@@ -337,41 +354,47 @@ def run_visual_language_generation_benchmark(model_path, framework, device, args
             for idx, input_text in enumerate(image_text_list):
                 p_idx = prompt_idx_list[idx]
                 if num == 0:
-                    metrics_print.print_unicode(f'[warm-up][P{p_idx}] Input text: {input_text}', max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
+                    metrics_print.print_unicode(f'[warm-up][P{p_idx}] Input text: {input_text}',
+                                                max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
                 iter_timestamp[num][p_idx]['start'] = datetime.datetime.now().isoformat()
                 gen_fn(
                     input_text, num, model, processor, args, iter_data_list, md5_list,
-                    p_idx, bench_hook, model_precision, proc_id, mem_consumption)
+                    p_idx, bench_hook, model_precision, proc_id, mem_consumption, required_frames)
                 iter_timestamp[num][p_idx]['end'] = datetime.datetime.now().isoformat()
-                prefix = '[warm-up]' if num == 0 else '[{}]'.format(num)
-                log.info(f"{prefix}[P{p_idx}] start: {iter_timestamp[num][p_idx]['start']}, end: {iter_timestamp[num][p_idx]['end']}")
+                prefix = f"[warm-up][P{p_idx}]" if num == 0 else f"[{num}][P{p_idx}]"
+                log.info(f"{prefix} start: {iter_timestamp[num][p_idx]['start']}, end: {iter_timestamp[num][p_idx]['end']}")
     else:
         for idx, input_text in enumerate(image_text_list):
             p_idx = prompt_idx_list[idx]
             for num in range(num_iters + 1):
                 if num == 0:
-                    metrics_print.print_unicode(f'[warm-up][P{p_idx}] Input text: {input_text}', max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
+                    metrics_print.print_unicode(f'[warm-up][P{p_idx}] Input text: {input_text}',
+                                                max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
                 iter_timestamp[num][p_idx]['start'] = datetime.datetime.now().isoformat()
                 gen_fn(
-                    input_text, num, model, processor, args, iter_data_list, md5_list,
-                    prompt_idx_list[idx], bench_hook, model_precision, proc_id, mem_consumption)
+                    input_text, num, model, processor, args, iter_data_list, md5_list, prompt_idx_list[idx],
+                    bench_hook, model_precision, proc_id, mem_consumption, required_frames)
                 iter_timestamp[num][p_idx]['end'] = datetime.datetime.now().isoformat()
-                prefix = '[warm-up]' if num == 0 else '[{}]'.format(num)
-                log.info(f"{prefix}[P{p_idx}] start: {iter_timestamp[num][p_idx]['start']}, end: {iter_timestamp[num][p_idx]['end']}")
+                prefix = f"[warm-up][P{p_idx}]" if num == 0 else f"[{num}][P{p_idx}]"
+                log.info(f"{prefix} start: {iter_timestamp[num][p_idx]['start']}, end: {iter_timestamp[num][p_idx]['end']}")
 
     metrics_print.print_average(iter_data_list, prompt_idx_list, args['batch_size'], True)
     return iter_data_list, pretrain_time, iter_timestamp
 
 
 def get_image_text_prompt(args):
     vlm_file_list = []
-    output_data_list, is_json_data = model_utils.get_param_from_file(args, ['media', "prompt"])
+    output_data_list, is_json_data = model_utils.get_param_from_file(args, ["media", "prompt"])
     if is_json_data:
         vlm_param_list = parse_json_data.parse_vlm_json_data(output_data_list)
         if len(vlm_param_list) > 0:
             for vlm_file in vlm_param_list:
-                if args['prompt_file'] is not None and len(args['prompt_file']) > 0:
-                    vlm_file['media'] = model_utils.resolve_media_file_path(vlm_file.get("media"), args['prompt_file'][0])
+                if args['prompt_file'] is not None and len(args['prompt_file']) > 0 and 'media' in vlm_file:
+                    if 'video' in vlm_file:
+                        raise ValueError('media and video cannot be specify in a single prompt file')
+                    vlm_file['media'] = model_utils.resolve_media_file_path(vlm_file.get('media'), args['prompt_file'][0])
+                elif args['prompt_file'] is not None and len(args['prompt_file']) > 0 and 'video' in vlm_file:
+                    vlm_file['video'] = model_utils.resolve_media_file_path(vlm_file.get('video'), args['prompt_file'][0])
                 vlm_file_list.append(vlm_file)
     else:
         vlm_file_list.append(output_data_list)