After review

Author: krzyczar

tools/llm_bench/llm_bench_utils/prompt_utils.py

@@ -10,7 +10,7 @@
 import logging as log
 from .model_utils import get_param_from_file
 from .parse_json_data import parse_text_json_data
-
+import openvino as ov
 
 def get_text_prompt(args):
     text_list = []
@@ -32,13 +32,16 @@ def inner(video_path, decym_frames):
             log.info(f"Requested to reduce into {decym_frames} frames")
         out_frames = func(video_path, decym_frames)
         log.info(f"Final frames number: {len(out_frames)}")
-        return np.array(out_frames)
+        log.info(f"First frame shape: {out_frames[0].shape}")
+        log.info(f"First frame dtype: {out_frames[0].dtype}")
+        return [ov.Tensor(frame) for frame in out_frames]
+        # return out_frames
     return inner
 
 
 @print_video_frames_number_and_convert_to_tensor
 def make_video_tensor(video_path, decym_frames=None):
-    supported_files = set([".mp4"])
+    supported_files = {".mp4"}
 
     assert os.path.exists(video_path), f"no input video file: {video_path}"
     assert video_path.suffix.lower() in supported_files, "no supported video file"
@@ -49,21 +52,24 @@ def make_video_tensor(video_path, decym_frames=None):
         ret, frame = cap.read()
         if not ret:
             break
+
         frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
         pil_image = Image.fromarray(frame_rgb)
+        np_img_array = np.array(pil_image)
+        output_frames.append(np_img_array)
 
-        shape = np.array(pil_image).shape
-        dtype = np.array(pil_image).dtype
-        log.info(f"Video shape: {shape}")
-        log.info(f"Video dtype: {dtype}")
-        new_frame = np.zeros(shape, dtype)
+        shape = np_img_array.shape
+        dtype = np_img_array.dtype
+        log.debug(f"Video shape: {shape}")
+        log.debug(f"Video dtype: {dtype}")
 
-        width, height = pil_image.size
-        log.info(f"Video size: {width}x{height}")
-        for x in range(0, width):
-            for y in range(0, height):
-                new_frame[y, x] = frame_rgb[y, x]
-        output_frames.append(np.array(pil_image))
+        # new_frame = np.zeros(shape, dtype=int)
+        # width, height = pil_image.size
+        # log.debug(f"Video size: {width}x{height}")
+        # for x in range(0, width):
+        #     for y in range(0, height):
+        #         new_frame[y, x] = frame_rgb[y, x]
+        # output_frames.append(new_frame)
 
     if decym_frames is None:
         return output_frames

tools/llm_bench/task/visual_language_generation.py

@@ -17,7 +17,7 @@
 import llm_bench_utils.output_file
 import llm_bench_utils.gen_output_data as gen_output_data
 import llm_bench_utils.parse_json_data as parse_json_data
-import llm_bench_utils.prompt_utils as pu
+import llm_bench_utils.prompt_utils as prompt_utils
 from pathlib import Path
 
 FW_UTILS = {'pt': llm_bench_utils.pt_utils, 'ov': llm_bench_utils.ov_utils}
@@ -33,32 +33,22 @@ def run_visual_language_generation_optimum(
     if args['batch_size'] != 1:
         log.warning("Only batch size 1 available for benchmarking")
         args["batch_size"] = 1
-    images = []
-    prompts = []
-    videos = []
-    inputs = [inputs] if not isinstance(inputs, (list, tuple)) else inputs
-    for input_data in inputs:
-        if input_data.get("video", None):
-            entry = Path(input_data["video"])
-            video_tensor = pu.make_video_tensor(entry, required_frames)
-            videos.append(video_tensor)
-        elif input_data.get("media", None):
-            entry = Path(input_data["media"])
-            if entry.is_dir():
-                for file in sorted(entry.iterdir()):
-                    images.append(load_image(str(file)))
-            else:
-                images.append(load_image(input_data["media"]))
-        prompts.append(input_data["prompt"])
-    prefix = '[warm-up]' if num == 0 else '[{}]'.format(num)
-    log.info(f'{prefix}[P{prompt_index}] Input image nums:{len(images)}')
+
     if args["output_dir"] is not None and num == 0:
         for bs_index, in_text in enumerate(prompts):
-            llm_bench_utils.output_file.output_input_text(in_text, args, model_precision, prompt_index, bs_index, proc_id)
+            llm_bench_utils.output_file.output_input_text(
+                in_text, args, model_precision,
+                prompt_index, bs_index, proc_id)
     tok_encode_start = time.perf_counter()
+
+    prompts, images, videos = extract_prompt_issues(inputs, required_frames)
+    prefix = '[warm-up]' if num == 0 else '[{}]'.format(num)
+    log.info(f'{prefix}[P{prompt_index}] Input image nums: {len(images)}')
+    log.info(f'{prefix}[P{prompt_index}] Input video nums: {len(videos)}')
     input_data = model.preprocess_inputs(text=prompts[0], image=images[0] if images else None, **processor)
     if videos:
         input_data["videos"] = videos
+
     tok_encode_end = time.perf_counter()
     tok_encode_time = (tok_encode_end - tok_encode_start) * 1000
     # Remove `token_type_ids` from inputs
@@ -201,26 +191,12 @@ def run_visual_language_generation_genai(
     if args['batch_size'] != 1:
         log.warning("Only batch size 1 available for benchmarking")
         args["batch_size"] = 1
-    images = []
-    prompts = []
-    videos = []
-    inputs = [inputs] if not isinstance(inputs, (list, tuple)) else inputs
-    for input_data in inputs:
-        if input_data.get("video", None):
-            entry = Path(input_data["video"])
-            video_tensor = pu.make_video_tensor(entry, required_frames)
-            videos.append(video_tensor)
-        elif input_data.get("media", None):
-            entry = Path(input_data["media"])
-            if entry.is_dir():
-                for file in sorted(entry.iterdir()):
-                    images.append(load_image_genai(str(file)))
-            else:
-                images.append(load_image_genai(input_data["media"]))
-        prompts.append(input_data["prompt"])
+
     if args["output_dir"] is not None and num == 0:
         for bs_index, in_text in enumerate(prompts):
-            llm_bench_utils.output_file.output_input_text(in_text, args, model_precision, prompt_index, bs_index, proc_id)
+            llm_bench_utils.output_file.output_input_text(
+                in_text, args, model_precision,
+                prompt_index, bs_index, proc_id)
     max_rss_mem_consumption = ''
     max_sys_mem_consumption = ''
     max_rss_mem_increase = ''
@@ -233,13 +209,18 @@ def run_visual_language_generation_genai(
     gen_config.num_beams = args["num_beams"]
     gen_config.do_sample = False
     gen_config.ignore_eos = True
+
     kwargs = {}
+    prompts, images, videos = extract_prompt_issues(inputs, required_frames)
+    prefix = '[warm-up]' if num == 0 else '[{}]'.format(num)
+    log.info(f'{prefix}[P{prompt_index}] Input image nums: {len(images)}')
+    log.info(f'{prefix}[P{prompt_index}] Input video nums: {len(videos)}')
+
     if images:
         kwargs["images"] = images
     if videos:
         kwargs["videos"] = videos
-    prefix = '[warm-up]' if num == 0 else '[{}]'.format(num)
-    log.info(f'{prefix}[P{prompt_index}] Input image nums:{len(images)}')
+
     start = time.perf_counter()
     generation_result = model.generate(prompts[0], generation_config=gen_config, **kwargs)
     end = time.perf_counter()
@@ -354,8 +335,8 @@ def run_visual_language_generation_benchmark(
             for idx, input_text in enumerate(image_text_list):
                 p_idx = prompt_idx_list[idx]
                 if num == 0:
-                    metrics_print.print_unicode(f'[warm-up][P{p_idx}] Input text: {input_text}',
-                                                max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
+                    prefix = f'[warm-up][P{p_idx}] Input text: {input_text}'
+                    metrics_print.print_unicode(prefix, max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
                 iter_timestamp[num][p_idx]['start'] = datetime.datetime.now().isoformat()
                 gen_fn(
                     input_text, num, model, processor, args, iter_data_list, md5_list,
@@ -368,8 +349,8 @@ def run_visual_language_generation_benchmark(
             p_idx = prompt_idx_list[idx]
             for num in range(num_iters + 1):
                 if num == 0:
-                    metrics_print.print_unicode(f'[warm-up][P{p_idx}] Input text: {input_text}',
-                                                max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
+                    prefix = f'[warm-up][P{p_idx}] Input text: {input_text}'
+                    metrics_print.print_unicode(prefix, max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
                 iter_timestamp[num][p_idx]['start'] = datetime.datetime.now().isoformat()
                 gen_fn(
                     input_text, num, model, processor, args, iter_data_list, md5_list, prompt_idx_list[idx],
@@ -382,6 +363,31 @@ def run_visual_language_generation_benchmark(
     return iter_data_list, pretrain_time, iter_timestamp
 
 
+def extract_prompt_issues(inputs, required_frames):
+    prompts, images, videos = [], [], []
+    if not isinstance(inputs, (list, tuple, set)):
+        inputs = [inputs]
+    for input_data in inputs:
+        if input_data.get("video") is not None:
+            entry = Path(input_data["video"])
+            if entry.is_dir():
+                for filename in sorted(entry.iterdir()):
+                    video_tensor = prompt_utils.make_video_tensor(filename, required_frames)
+                    videos.append(video_tensor)
+            else:
+                video_tensor = prompt_utils.make_video_tensor(entry, required_frames)
+                videos.append(video_tensor)
+        if input_data.get("media") is not None:
+            entry = Path(input_data["media"])
+            if entry.is_dir():
+                for file in sorted(entry.iterdir()):
+                    images.append(load_image(str(file)))
+            else:
+                images.append(load_image(str(entry)))
+        prompts.append(input_data["prompt"])
+    return prompts, images, videos
+
+
 def get_image_text_prompt(args):
     vlm_file_list = []
     output_data_list, is_json_data = model_utils.get_param_from_file(args, ["media", "prompt"])
@@ -393,7 +399,7 @@ def get_image_text_prompt(args):
                     if 'video' in vlm_file:
                         raise ValueError('media and video cannot be specify in a single prompt file')
                     vlm_file['media'] = model_utils.resolve_media_file_path(vlm_file.get('media'), args['prompt_file'][0])
-                elif args['prompt_file'] is not None and len(args['prompt_file']) > 0 and 'video' in vlm_file:
+                if args['prompt_file'] is not None and len(args['prompt_file']) > 0 and 'video' in vlm_file:
                     vlm_file['video'] = model_utils.resolve_media_file_path(vlm_file.get('video'), args['prompt_file'][0])
                 vlm_file_list.append(vlm_file)
     else: