+ tests

Author: krzyczar

tests/python_tests/samples/test_tools_llm_benchmark.py

@@ -27,6 +27,12 @@
     {"steps": 30, "width": 64, "height": 128, "guidance_scale": 1.0, "strength": "0.8", "media": "cat.png", "prompt": image_generation_i2i_prompt},
 ]
 
+# @pytest.fixture(scope="module")
+# def real_video(pytestconfig):
+#     spinning_earth_url = TEST_VIDEO_URLS["spinning-earth-480"]
+#     video = from_cache_or_download(pytestconfig, spinning_earth_url, "spinning-earth-480.mp4")
+#     return video
+
 class TestBenchmarkLLM:
     @pytest.mark.samples
     @pytest.mark.parametrize(

tools/llm_bench/benchmark.py

@@ -231,7 +231,7 @@ def get_argprser():
     parser.add_argument("--vocoder_path", type=str, default=None,
                         help="Path to vocoder  for text to speech scenarios")
     parser.add_argument("-vf", "--video_frames", type=int, default=None,
-                        help="controller of video frames to process (required frame number or decymation factor if negative)")
+                        help="controller of video frames to process (required frame number if positive or decymation factor if negative)")
     return parser.parse_args()
 
 
@@ -316,10 +316,6 @@ def main():
             iter_data_list, pretrain_time, iter_timestamp = CASE_TO_BENCH[model_args['use_case'].task](
                 model_path, framework, args.device, args.tokens_len, args.streaming, model_args,
                 args.num_iters, memory_data_collector)
-        elif model_args['use_case'].task == "visual_text_gen":
-            iter_data_list, pretrain_time, iter_timestamp = CASE_TO_BENCH[model_args['use_case'].task](
-                model_path, framework, args.device, model_args, args.num_iters,
-                memory_data_collector, decym_frames=args.video_frames)
         else:
             iter_data_list, pretrain_time, iter_timestamp = CASE_TO_BENCH[model_args['use_case'].task](
                 model_path, framework, args.device, model_args, args.num_iters, memory_data_collector)

tools/llm_bench/llm_bench_utils/model_utils.py

@@ -142,7 +142,7 @@ def analyze_args(args):
     model_args["rerank_texts"] = args.texts
     model_args["rerank_texts_file"] = args.texts_file
     model_args["apply_chat_template"] = args.apply_chat_template
-
+    model_args["video_frames"] = args.video_frames
     optimum = args.optimum
 
     if optimum and args.genai:

tools/llm_bench/llm_bench_utils/prompt_utils.py

@@ -15,6 +15,7 @@
 from .parse_json_data import parse_vlm_json_data
 from pathlib import Path
 import openvino as ov
+import math
 
 
 def get_text_prompt(args):
@@ -87,15 +88,20 @@ def make_video_tensor(video_path, decym_frames=None):
         return output_frames
 
     # decymation procedure
-    # decym_fames is required max frame number if positive
+    # decym_frames is required max frame number if positive
     # or decymation factor if negative
+    # e.g if input frames number is 100 and decym_fames = 5:
+    #        then number of processed frames are: 0, 20, 40, 60, 80
+    #     if input frames number is 100 and decym_fames = -5:
+    #        then number of processed frames are: 0, 5, 10, 15, 20, ...
 
     decym_frames = int(decym_frames)
     if decym_frames > 0:
         if len(output_frames) <= decym_frames:
             log.info(f"Video decym: too short to decym: crop: {decym_frames}")
             return list(output_frames[:decym_frames])
-        decym_factor = 1 + int(len(output_frames) / decym_frames)
+        decym_factor_f = float(len(output_frames)) / decym_frames
+        decym_factor = int(math.ceil(decym_factor_f))
     else:
         decym_factor = -decym_frames
     log.info(f"Video decym factor: {decym_factor}")

tools/llm_bench/task/visual_language_generation.py

@@ -27,13 +27,14 @@
 
 def run_visual_language_generation_optimum(
         inputs, num, model, processor, args, iter_data_list, md5_list, prompt_index,
-        bench_hook, model_precision, proc_id, mem_consumption, decym_frames=None):
+        bench_hook, model_precision, proc_id, mem_consumption):
     from optimum.intel.utils.import_utils import is_transformers_version
     set_seed(args['seed'])
     if args['batch_size'] != 1:
         log.warning("Only batch size 1 available for benchmarking")
         args["batch_size"] = 1
 
+    decym_frames = args["video_frames"]
     prompts, images, videos = extract_prompt_issues(inputs, decym_frames, False)
     if args["output_dir"] is not None and num == 0:
         for bs_index, in_text in enumerate(prompts):
@@ -180,12 +181,13 @@ def run_visual_language_generation_optimum(
 
 
 def run_visual_language_generation_genai(
-        inputs, num, model, processor, args, iter_data_list, md5_list, prompt_index,
-        streamer, model_precision, proc_id, mem_consumption, decym_frames=None):
+        inputs, num, model, processor, args, iter_data_list, md5_list,
+        prompt_index, streamer, model_precision, proc_id, mem_consumption):
     if args['batch_size'] != 1:
         log.warning("Only batch size 1 available for benchmarking")
         args["batch_size"] = 1
 
+    decym_frames = args["video_frames"]
     prompts, images, videos = extract_prompt_issues(inputs, decym_frames, True)
     if args["output_dir"] is not None and num == 0:
         for bs_index, in_text in enumerate(prompts):
@@ -294,9 +296,7 @@ def run_visual_language_generation_genai(
         metrics_print.print_generated(num, warm_up=(num == 0), generated=generated_text[0], prompt_idx=prompt_index)
 
 
-def run_visual_language_generation_benchmark(
-        model_path, framework, device, args, num_iters,
-        mem_consumption, decym_frames=None):
+def run_visual_language_generation_benchmark(model_path, framework, device, args, num_iters, mem_consumption):
     outs = FW_UTILS[framework].create_image_text_gen_model(model_path, device, mem_consumption, **args)
     model, processor, pretrain_time, bench_hook, use_genai = outs
     model_precision = model_utils.get_model_precision(model_path.parts)
@@ -335,7 +335,7 @@ def run_visual_language_generation_benchmark(
                 iter_timestamp[num][p_idx]['start'] = datetime.datetime.now().isoformat()
                 gen_fn(
                     input_text, num, model, processor, args, iter_data_list, md5_list,
-                    p_idx, bench_hook, model_precision, proc_id, mem_consumption, decym_frames)
+                    p_idx, bench_hook, model_precision, proc_id, mem_consumption)
                 iter_timestamp[num][p_idx]['end'] = datetime.datetime.now().isoformat()
                 prefix = f"[warm-up][P{p_idx}]" if num == 0 else f"[{num}][P{p_idx}]"
                 log.info(f"{prefix} start: {iter_timestamp[num][p_idx]['start']}, end: {iter_timestamp[num][p_idx]['end']}")
@@ -348,8 +348,8 @@ def run_visual_language_generation_benchmark(
                     metrics_print.print_unicode(prefix, max_output=metrics_print.MAX_INPUT_TXT_IN_LOG)
                 iter_timestamp[num][p_idx]['start'] = datetime.datetime.now().isoformat()
                 gen_fn(
-                    input_text, num, model, processor, args, iter_data_list, md5_list, prompt_idx_list[idx],
-                    bench_hook, model_precision, proc_id, mem_consumption, decym_frames)
+                    input_text, num, model, processor, args, iter_data_list, md5_list,
+                    prompt_idx_list[idx], bench_hook, model_precision, proc_id, mem_consumption)
                 iter_timestamp[num][p_idx]['end'] = datetime.datetime.now().isoformat()
                 prefix = f"[warm-up][P{p_idx}]" if num == 0 else f"[{num}][P{p_idx}]"
                 log.info(f"{prefix} start: {iter_timestamp[num][p_idx]['start']}, end: {iter_timestamp[num][p_idx]['end']}")