AC: fix input blob info (#3416)

* AC: fix input blob info

* properly enable input precision setting

* preprocessing setting

Author: eaidova

tools/accuracy_checker/openvino/tools/accuracy_checker/evaluators/model_evaluator.py

@@ -75,13 +75,17 @@ def from_configs(cls, model_config, delayed_annotation_loading=False):
             dataset_config.get('_ie_preprocessing', False)
             if launcher_config['framework'] == 'dlsdk' else False
         )
+        enable_ov_preprocessing = (
+            dataset_config.get('_ie_preprocessing', False)
+            if launcher_config['framework'] == 'openvino' else False
+        )
         preprocessor = PreprocessingExecutor(
             dataset_config.get('preprocessing'), dataset_name, dataset_metadata,
-            enable_ie_preprocessing=enable_ie_preprocessing
+            enable_ie_preprocessing=enable_ie_preprocessing, enable_ov_preprocessing=enable_ov_preprocessing
         )
         input_precision = launcher_config.get('_input_precision', [])
         input_layouts = launcher_config.get('_input_layout', '')
-        if enable_ie_preprocessing:
+        if enable_ie_preprocessing or enable_ov_preprocessing:
             launcher_kwargs['preprocessor'] = preprocessor
         if launcher_config['framework'] == 'dummy' and launcher_config.get('provide_identifiers', False):
             launcher_kwargs = {'identifiers': dataset.identifiers if dataset is not None else []}
@@ -705,12 +709,13 @@ def _resolve_undefined_shapes(self):
             if self.launcher.dyn_batch_only:
                 self.launcher.resolve_undefined_batch()
                 return
-            if (
-                    (self.preprocessor.dynamic_shapes or not self.preprocessor.has_shape_modifications)
-                    and self.launcher.dynamic_shapes_policy != 'static'):
-                self._initialize_input_shape_with_data_range()
-                return
-            self._initialize_input_shape()
+            if getattr(self.launcher, 'need_dyn_resolving', True):
+                if (
+                        (self.preprocessor.dynamic_shapes or not self.preprocessor.has_shape_modifications)
+                        and self.launcher.dynamic_shapes_policy != 'static'):
+                    self._initialize_input_shape_with_data_range()
+                    return
+                self._initialize_input_shape()
 
     def _initialize_input_shape_with_data_range(self):
         input_shapes = []

tools/accuracy_checker/openvino/tools/accuracy_checker/launcher/openvino_launcher.py

@@ -22,6 +22,7 @@
 import warnings
 import numpy as np
 from openvino.runtime import Core, AsyncInferQueue, get_version, PartialShape, Type, Dimension
+from openvino.preprocess import PrePostProcessor
 from .dlsdk_launcher_config import (
     HETERO_KEYWORD, MULTI_DEVICE_KEYWORD, NIREQ_REGEX, VPU_PLUGINS,
     get_cpu_extension,
@@ -499,7 +500,7 @@ def _create_network(self, input_shapes=None):
             with open(str(self._model), 'rb') as f: #pylint:disable=unspecified-encoding
                 self.exec_network = self.ie_core.import_model(io.BytesIO(f.read()), self._device)
             self.original_outputs = self.exec_network.outputs
-            model_batch = self._get_model_batch_size()
+            model_batch = self._get_remodel_batch_size()
             self._batch = model_batch if model_batch is not None else 1
             return
         if self._weights is None and self._model.suffix != '.onnx':
@@ -587,8 +588,10 @@ def load_network(self, network=None, log=False, preprocessing=None):
             self.network = network
         if self.network is not None:
             self.dyn_input_layers, self._partial_shapes = self.get_dynamic_inputs(self.network)
-        self.input_to_tensor_name = self.get_input_tensor_name_mapping(self.network)
-        self.input_to_index = {inp.get_node().friendly_name: idx for idx, inp in enumerate(self.network.inputs)}
+        self.input_to_tensor_name = self.get_input_tensor_name_mapping(
+            self.network if self.network is not None else self.exec_network)
+        network_inputs = self.network.inputs if self.network is not None else self.exec_network.inputs
+        self.input_to_index = {inp.get_node().friendly_name: idx for idx, inp in enumerate(network_inputs)}
         if not self._postpone_input_configuration:
             self._set_precision()
             self._set_input_shape()
@@ -597,12 +600,13 @@ def load_network(self, network=None, log=False, preprocessing=None):
                 self.print_input_output_info(self.network if self.network is not None else self.exec_network)
             if preprocessing:
                 self._set_preprocess(preprocessing)
+                self.dyn_input_layers, self._partial_shapes = self.get_dynamic_inputs(self.network)
             model_batch = self._get_model_batch_size()
             model_batch = 1 if model_batch is None else model_batch
             self._batch = self.config.get('batch', model_batch)
             self._set_batch_size(self._batch)
             self.try_to_set_default_layout()
-            if self.network and not preprocessing and (not self.dyn_input_layers or self.is_dynamic):
+            if self.network and (not self.dyn_input_layers or self.is_dynamic or self.disable_resize_to_input):
                 self.exec_network = self.ie_core.compile_model(self.network, self._device)
                 self.infer_request = self.exec_network.create_infer_request()
 
@@ -838,11 +842,15 @@ def _data_to_blob(self, layer_shape, data, layout):  # pylint:disable=R0911,R091
 
     def _set_precision(self):
         config_inputs = self.config.get('inputs', [])
-        for input_config in config_inputs:
-            if 'precision' in input_config:
-                if self.network:
-                    self.inputs[input_config['name']].set_element_type(
-                        PRECISION_STR_TO_TYPE[input_config['precision'].upper()])
+        has_precisions = ['precision' in inp for inp in config_inputs]
+        if has_precisions and self.network:
+            preprocessor = PrePostProcessor(self.network)
+            for input_config in config_inputs:
+                if 'precision' in input_config:
+                    name = input_config['name']
+                    element_type =  PRECISION_STR_TO_TYPE[input_config['precision'].upper()]
+                    preprocessor.input(self.input_to_index[name]).tensor().set_element_type(element_type)
+            self.network = preprocessor.build()
 
     def _set_input_shape(self):
         if not self.network:
@@ -914,33 +922,28 @@ def _set_preprocess(self, preprocess):
             preprocess_steps = preprocess.ie_preprocess_steps
             if not preprocess_steps:
                 return
-            for input_name, input_info in self.network.input_info.items():
+            preprocessor = PrePostProcessor(self.network)
+            for input_name in self.inputs:
                 if input_name in self.const_inputs + self.image_info_inputs:
                     continue
+                input_id = self.input_to_index[input_name]
                 for (name, value) in preprocess_steps:
-                    setattr(input_info.preprocess_info, name, value)
-                if preprocess.ie_processor.has_normalization():
-                    channel_id = input_info.layout.find('C')
-                    if channel_id != -1:
-                        num_channels = input_info.input_data.shape[channel_id]
-                        preprocess.ie_processor.set_normalization(num_channels, input_info.preprocess_info)
-            self.disable_resize_to_input = preprocess.ie_processor.has_resize()
-            self._use_set_blob = self.disable_resize_to_input
-            self.load_network(self.network)
+                    if name == 'resize_algorithm':
+                        preprocessor.input(input_id).tensor().set_spatial_dynamic_shape()
+                        preprocessor.input(input_id).preprocess().resize(value)
+                        self.need_dyn_resolving = False
+                    if name == 'convert_color_format':
+                        src, dst = value
+                        preprocessor.input(input_id).tensor().set_color_format(src)
+                        preprocessor.input(input_id).preprocess().convert_color(dst)
+                    if name == 'mean_variant':
+                        mean, scale = value
+                        if mean is not None:
+                            preprocessor.input(input_id).preprocess().mean(mean)
+                        if scale is not None:
+                            preprocessor.input(input_id).preprocess().scale(scale)
+            self.network = preprocessor.build()
             self._preprocess_steps = preprocess_steps
-            return
-        preprocess_info_by_input = {}
-        preprocess_info = preprocess.preprocess_info
-        for input_name in self.inputs:
-            if input_name in self.const_inputs + self.image_info_inputs:
-                continue
-            if preprocess.ie_processor.has_normalization():
-                channel_id = self.inputs[input_name].layout.find('C')
-                if channel_id != -1:
-                    num_channels = self.inputs[input_name].shape[channel_id]
-                    preprocess.ie_processor.set_normalization(num_channels, preprocess_info)
-            preprocess_info_by_input[input_name] = preprocess_info
-        self._preprocess_info = preprocess_info_by_input
         self.disable_resize_to_input = preprocess.ie_processor.has_resize()
 
     def get_model_file_type(self):

tools/accuracy_checker/openvino/tools/accuracy_checker/preprocessor/ov_preprocessor.py

@@ -0,0 +1,125 @@
+"""
+Copyright (c) 2018-2022 Intel Corporation
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+from copy import deepcopy
+from collections import namedtuple
+import warnings
+try:
+    from openvino.preprocess import ResizeAlgorithm, ColorFormat # pylint:disable=W9902
+except ImportError:
+    ResizeAlgorithm = None
+from ..utils import get_or_parse_value
+
+
+def ov_preprocess_available():
+    return ResizeAlgorithm is not None
+
+
+PreprocessingOp = namedtuple('PreprocessingOp', ['name', 'value'])
+
+
+class OVPreprocessor:
+    PRECOMPUTED_MEANS = {
+        'imagenet': (104.00698793, 116.66876762, 122.67891434),
+        'cifar10': (125.307, 122.961, 113.8575),
+    }
+
+    PRECOMPUTED_STDS = {
+        'imagenet': (104.00698793, 116.66876762, 122.67891434),
+        'cifar10': (125.307, 122.961, 113.8575),
+    }
+
+    def __init__(self, config):
+        self.SUPPORTED_PREPROCESSING_OPS = {
+            'resize': self.get_resize_op,
+            'auto_resize': self.get_resize_op,
+            'bgr_to_rgb': self.get_color_format_op,
+            'rgb_to_bgr': self.get_color_format_op,
+            'nv12_to_bgr': self.get_color_format_op,
+            'nv12_to_rgb': self.get_color_format_op,
+            'normalization': self.get_normalization_op
+        }
+        self.mean_values = None
+        self.std_values = None
+        self.config = config or []
+        self.configure()
+
+    def configure(self):
+        steps = []
+        step_names = set()
+        keep_preprocessing_config = deepcopy(self.config)
+        for preprocessor in reversed(self.config):
+            if preprocessor['type'] not in self.SUPPORTED_PREPROCESSING_OPS:
+                break
+            ie_ops = self.get_op(preprocessor)
+            if not ie_ops:
+                break
+            if ie_ops.name in step_names:
+                break
+            step_names.add(ie_ops.name)
+            steps.append(ie_ops)
+            keep_preprocessing_config = keep_preprocessing_config[:-1]
+        self.steps = steps
+        self.keep_preprocessing_info = keep_preprocessing_config
+        if not steps:
+            warnings.warn('no preprocessing steps for transition to PreProcessInfo')
+
+    def get_op(self, preprocessing_config):
+        preprocessing_getter = self.SUPPORTED_PREPROCESSING_OPS[preprocessing_config['type']]
+        return preprocessing_getter(preprocessing_config)
+
+    @staticmethod
+    def get_resize_op(config):
+        supported_interpolations = {
+            'LINEAR': ResizeAlgorithm.RESIZE_LINEAR,
+            'BILINEAR': ResizeAlgorithm.RESIZE_LINEAR,
+            'NEAREST': ResizeAlgorithm.RESIZE_NEAREST,
+            'CUBIC': ResizeAlgorithm.RESIZE_CUBIC
+        }
+        if 'aspect_ratio' in config:
+            return None
+        interpolation = config.get('interpolation', 'BILINEAR').upper()
+        if interpolation not in supported_interpolations:
+            return None
+        return PreprocessingOp('resize_algorithm', supported_interpolations[interpolation])
+
+    @staticmethod
+    def get_color_format_op(config):
+        supported_color_formats = {
+            'bgr': ColorFormat.BGR,
+            'rgb': ColorFormat.RGB,
+            'nv12': ColorFormat.NV12_SINGLE_PLANE
+        }
+        src_format = config['type'].split('_')[0]
+        dst_format = config['type'].split('_')[-1]
+        src_color = supported_color_formats.get(src_format)
+        dst_color = supported_color_formats.get(dst_format)
+        if dst_color is None or src_color is None:
+            return None
+        return PreprocessingOp('convert_color_format', (src_color, dst_color))
+
+    def get_normalization_op(self, config):
+        mean_values = get_or_parse_value(config.get('mean'), self.PRECOMPUTED_MEANS)
+        std_values = get_or_parse_value(config.get('std'), self.PRECOMPUTED_STDS)
+        return PreprocessingOp('mean_variant', (mean_values, std_values))
+
+    def has_resize(self):
+        preprocessor_names = [step.name for step in self.steps]
+        return 'resize_algorithm' in preprocessor_names
+
+    def has_normalization(self):
+        preprocessor_names = [step.name for step in self.steps]
+        return 'mean_variant' in preprocessor_names

tools/accuracy_checker/openvino/tools/accuracy_checker/preprocessor/preprocessing_executor.py

@@ -19,13 +19,15 @@
 from ..utils import get_data_shapes
 from .preprocessor import Preprocessor, MULTI_INFER_PREPROCESSORS
 from .ie_preprocessor import IEPreprocessor, ie_preprocess_available
+from .ov_preprocessor import OVPreprocessor, ov_preprocess_available
 
 
 class PreprocessingExecutor:
     def __init__(
             self, processors=None, dataset_name='custom', dataset_meta=None,
             input_shapes=None,
-            enable_ie_preprocessing=False
+            enable_ie_preprocessing=False,
+            enable_ov_preprocessing=False
     ):
         self.processors = []
         self.dataset_meta = dataset_meta
@@ -40,6 +42,15 @@ def __init__(
             else:
                 self.ie_processor = IEPreprocessor(processors)
                 processors = self.ie_processor.keep_preprocessing_info
+        if enable_ov_preprocessing:
+            if not ov_preprocess_available():
+                warnings.warn(
+                    'PreProcessInfo is not available in your InferenceEngine version or openvino is not installed'
+                    '--ie_preprocessing key will be ignored'
+                )
+            else:
+                self.ie_processor = OVPreprocessor(processors)
+                processors = self.ie_processor.keep_preprocessing_info
 
         if not processors:
             return