After review

Author: krzyczar

tools/llm_bench/benchmark.py

@@ -78,6 +78,7 @@ def get_argprser():
         'if the value equals 0 (default), execute the warm-up iteration(0th iteration).',
     )
     parser.add_argument('-i', '--images', default=None, help='test images for vision tasks. Can be directory or path to single image')
+    parser.add_argument('-vp', '--videos', default=None, help='test videos for vision tasks. Can be directory or path to single video')
     parser.add_argument('-s', '--seed', type=int, default=42, required=False, help='specific random seed to generate fix result. Default 42.')
     parser.add_argument(
         '-lc',
@@ -235,11 +236,11 @@ def get_argprser():
 
 
 CASE_TO_BENCH = {
-    'text_gen': bench_text.run_text_generation_benchmark,
-    'image_gen': bench_image.run_image_generation_benchmark,
-    'code_gen': bench_text.run_text_generation_benchmark,
-    'ldm_super_resolution': bench_ldm_sr.run_ldm_super_resolution_benchmark,
-    'speech_to_text': bench_speech.run_speech_2_txt_benchmark,
+    "text_gen": bench_text.run_text_generation_benchmark,
+    "image_gen": bench_image.run_image_generation_benchmark,
+    "code_gen": bench_text.run_text_generation_benchmark,
+    "ldm_super_resolution": bench_ldm_sr.run_ldm_super_resolution_benchmark,
+    "speech_to_text": bench_speech.run_speech_2_txt_benchmark,
     "visual_text_gen": bench_vlm.run_visual_language_generation_benchmark,
     "text_embed": bench_text_embed.run_text_embddings_benchmark,
     "text_to_speech": bench_text_to_speech.run_text_2_speech_benchmark,
@@ -318,7 +319,7 @@ def main():
         else:
             iter_data_list, pretrain_time, iter_timestamp = CASE_TO_BENCH[model_args['use_case'].task](
                 model_path, framework, args.device, model_args, args.num_iters,
-                memory_data_collector, args.video_frames)
+                memory_data_collector, decym_frames=args.video_frames)
         if args.report is not None or args.report_json is not None:
             model_precision = ''
             if framework == 'ov':

tools/llm_bench/llm_bench_utils/model_utils.py

@@ -43,6 +43,7 @@ def get_param_from_file(args, input_key):
 
             elif args[input_key] is not None and args['prompt_file'] is not None:
                 raise RuntimeError(f'== {input_key} and prompt file should not exist together ==')
+
             else:
                 if args[input_key] is not None:
                     if args[input_key] != '':
@@ -56,11 +57,16 @@ def get_param_from_file(args, input_key):
             if "media" in input_key:
                 if args["media"] is None and args["images"] is None:
                     if args["use_case"].task == "visual_text_gen":
-                        log.warn("Input image is not provided. Only text generation part will be evaluated")
+                        if args["videos"] is None:
+                            log.warn("Input image/video is not provided. Only text generation part will be evaluated")
                     elif args["use_case"].task != "image_gen":
                         raise RuntimeError("No input image. ImageToImage/Inpainting Models cannot start generation without one. Please, provide an image.")
                 else:
                     data_dict["media"] = args["media"] if args["media"] is not None else args["images"]
+            if "video" in input_key:
+                if args["videos"] is not None:
+                    data_dict["video"] = args["videos"]
+
             if args["prompt"] is None:
                 if args["use_case"].task == "visual_text_gen":
                     data_dict["prompt"] = "What is OpenVINO?" if data_dict.get("media") is None else "Describe image"
@@ -112,6 +118,7 @@ def analyze_args(args):
     model_args["height"] = args.height
     model_args["width"] = args.width
     model_args['images'] = args.images
+    model_args['videos'] = args.videos
     model_args['seed'] = args.seed
     model_args['mem_consumption'] = args.memory_consumption
     model_args['batch_size'] = args.batch_size

tools/llm_bench/llm_bench_utils/parse_json_data.py

@@ -27,7 +27,7 @@ def parse_vlm_json_data(json_data_list):
     for json_data in json_data_list:
         prompt_data = create_base_prompt(json_data)
         if ("media" in json_data) and ("video" in json_data):
-            raise ValueError("only one key is avaialble from media & video")
+            raise ValueError("only one key is available from media & video")
         if "media" in json_data:
             prompt_data["media"] = json_data["media"]
         if "video" in json_data:

tools/llm_bench/llm_bench_utils/prompt_utils.py

@@ -8,8 +8,13 @@
 import numpy as np
 from PIL import Image
 import logging as log
+from transformers.image_utils import load_image
 from .model_utils import get_param_from_file
+from .model_utils import resolve_media_file_path
 from .parse_json_data import parse_text_json_data
+from .parse_json_data import parse_vlm_json_data
+from pathlib import Path
+import openvino as ov
 
 
 def get_text_prompt(args):
@@ -26,19 +31,24 @@ def get_text_prompt(args):
 
 
 def print_video_frames_number_and_convert_to_tensor(func):
-    def inner(video_path, decym_frames):
+    def inner(video_path, genai_flag, decym_frames):
         log.info(f"Input video file: {video_path}")
         if decym_frames is not None:
             log.info(f"Requested to reduce into {decym_frames} frames")
-        out_frames = func(video_path, decym_frames)
+        out_frames = func(video_path, genai_flag, decym_frames)
         log.info(f"Final frames number: {len(out_frames)}")
-        return np.array(out_frames)
+        log.info(f"First frame shape: {out_frames[0].shape}")
+        log.info(f"First frame dtype: {out_frames[0].dtype}")
+        if genai_flag:
+            return [ov.Tensor(frame[None]) for frame in out_frames]
+        else:
+            return np.array(out_frames)
     return inner
 
 
 @print_video_frames_number_and_convert_to_tensor
-def make_video_tensor(video_path, decym_frames=None):
-    supported_files = set([".mp4"])
+def make_video_tensor(video_path, genai_flag, decym_frames=None):
+    supported_files = {".mp4"}
 
     assert os.path.exists(video_path), f"no input video file: {video_path}"
     assert video_path.suffix.lower() in supported_files, "no supported video file"
@@ -49,38 +59,89 @@ def make_video_tensor(video_path, decym_frames=None):
         ret, frame = cap.read()
         if not ret:
             break
+
         frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
         pil_image = Image.fromarray(frame_rgb)
 
-        shape = np.array(pil_image).shape
-        dtype = np.array(pil_image).dtype
-        log.info(f"Video shape: {shape}")
-        log.info(f"Video dtype: {dtype}")
-        new_frame = np.zeros(shape, dtype)
-
-        width, height = pil_image.size
-        log.info(f"Video size: {width}x{height}")
-        for x in range(0, width):
-            for y in range(0, height):
-                new_frame[y, x] = frame_rgb[y, x]
-        output_frames.append(np.array(pil_image))
+        np_img_array = np.array(pil_image)
+        log.debug(f"Video shape: {np_img_array.shape}")
+        log.debug(f"Video dtype: {np_img_array.dtype}")
+        output_frames.append(np_img_array)
 
     if decym_frames is None:
+        log.info("Video decym: none: skip")
         return output_frames
     if int(decym_frames) == 0:
+        log.info("Video decym: zero: skip")
         return output_frames
 
-    # decimation procedure
-    # decim_fames is required frame number if positive
-    # or decimation factor if negative
+    # decymation procedure
+    # decym_fames is required max frame number if positive
+    # or decymation factor if negative
 
     decym_frames = int(decym_frames)
     if decym_frames > 0:
         if len(output_frames) <= decym_frames:
-            return output_frames
-        decym_factor = int(len(output_frames) / decym_frames)
+            log.info(f"Video decym: too short to decym: crop: {decym_frames}")
+            return list(output_frames[:decym_frames])
+        decym_factor = 1 + int(len(output_frames) / decym_frames)
     else:
         decym_factor = -decym_frames
+    log.info(f"Video decym factor: {decym_factor}")
     if decym_factor >= 2:
-        return output_frames[::decym_factor]
+        return list(output_frames[::decym_factor])
+    log.info("Video decym: too large decym factor: skip")
     return output_frames
+
+
+def load_image_genai(image_path):
+    pil_image = load_image(image_path)
+    image_data = np.array(pil_image)[None]
+    return ov.Tensor(image_data)
+
+
+def extract_prompt_issues(inputs, genai_flag, required_frames):
+    prompts, images, videos = [], [], []
+    if not isinstance(inputs, (list, tuple, set)):
+        inputs = [inputs]
+    for input_data in inputs:
+        if input_data.get("video") is not None:
+            entry = Path(input_data["video"])
+            if entry.is_dir():
+                for filename in sorted(entry.iterdir()):
+                    video_tensor = make_video_tensor(filename, genai_flag, required_frames)
+                    videos.extend(video_tensor)
+            else:
+                video_tensor = make_video_tensor(entry, genai_flag, required_frames)
+                videos.extend(video_tensor)
+        if input_data.get("media") is not None:
+            func_load_image = load_image_genai if genai_flag else load_image
+            entry = Path(input_data["media"])
+            if entry.is_dir():
+                for file in sorted(entry.iterdir()):
+                    img = func_load_image(str(file))
+                    images.append(img)
+            else:
+                img = func_load_image(input_data["media"])
+                images.append(img)
+        prompts.append(input_data["prompt"])
+    return prompts, images, videos
+
+
+def get_image_text_prompt(args):
+    vlm_file_list = []
+    output_data_list, is_json_data = get_param_from_file(args, ["video", "media", "prompt"])
+    if is_json_data:
+        vlm_param_list = parse_vlm_json_data(output_data_list)
+        if len(vlm_param_list) > 0:
+            for vlm_file in vlm_param_list:
+                if args['prompt_file'] is not None and len(args['prompt_file']) > 0 and 'media' in vlm_file:
+                    if 'video' in vlm_file:
+                        raise ValueError('media and video cannot be specify in a single prompt file')
+                    vlm_file['media'] = resolve_media_file_path(vlm_file.get('media'), args['prompt_file'][0])
+                if args['prompt_file'] is not None and len(args['prompt_file']) > 0 and 'video' in vlm_file:
+                    vlm_file['video'] = resolve_media_file_path(vlm_file.get('video'), args['prompt_file'][0])
+                vlm_file_list.append(vlm_file)
+    else:
+        vlm_file_list.append(output_data_list)
+    return vlm_file_list