Perform final filtering on full, merged model (#1398)

* Validate server template settings for create and prepare tools, with complete model

* Added final filters to check dynamic servers and admin server for targets

Author: rakillen

core/src/main/python/create.py

@@ -35,6 +35,7 @@
 from wlsdeploy.tool.util.archive_helper import ArchiveHelper
 from wlsdeploy.tool.util.wlst_helper import WlstHelper
 from wlsdeploy.tool.util import wlst_helper
+from wlsdeploy.tool.validate.content_validator import ContentValidator
 from wlsdeploy.util import cla_helper
 from wlsdeploy.util import getcreds
 from wlsdeploy.util import tool_main
@@ -313,6 +314,10 @@ def main(model_context):
         model_dictionary = cla_helper.load_model(_program_name, model_context, aliases, "create", __wlst_mode,
                                                  validate_crd_sections=False)
 
+        # check for any content problems in the merged, substituted model
+        content_validator = ContentValidator(model_context)
+        content_validator.validate_model(model_dictionary)
+
         archive_helper = None
         archive_file_name = model_context.get_archive_file_name()
         if archive_file_name:

core/src/main/python/discover.py

@@ -498,6 +498,8 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject
     if filter_helper.apply_filters(model.get_model(), "discover", model_context):
         __logger.info('WLSDPLY-06014', _class_name=_class_name, method_name=_method_name)
 
+    filter_helper.apply_final_filters(model.get_model(), model.get_model(), model_context)
+
     # target config always present in model context, default config if not declared
     target_configuration = model_context.get_target_configuration()
 

core/src/main/python/wlsdeploy/tool/prepare/model_preparer.py

@@ -23,6 +23,7 @@
 from wlsdeploy.tool.util.credential_injector import CredentialInjector
 from wlsdeploy.tool.util.variable_injector import VARIABLE_FILE_UPDATE
 from wlsdeploy.tool.util.variable_injector import VariableInjector
+from wlsdeploy.tool.validate.content_validator import ContentValidator
 from wlsdeploy.tool.validate.validator import Validator
 from wlsdeploy.util import cla_helper
 from wlsdeploy.util import model
@@ -271,6 +272,21 @@ def _clean_archive_files(self):
         if self.model_context.get_target_configuration().exclude_domain_bin_contents():
             archive_helper.remove_domain_scripts()
 
+    def _apply_final_filters(self, model_dictionary):
+        """
+        Apply final filters to the merged model dictionary,
+        and apply any updates to the last model in the command-line list.
+        """
+        model_file_list = self.model_files.split(',')
+        last_file = model_file_list[-1]
+        update_file = os.path.join(self.output_dir, os.path.basename(last_file))
+
+        update_model_dict = FileToPython(update_file, True).parse()
+        filter_helper.apply_final_filters(model_dictionary, update_model_dict, self.model_context)
+
+        pty = PythonToYaml(update_model_dict)
+        pty.write_to_yaml_file(update_file)
+
     def prepare_models(self):
         """
         Replace password attributes in each model file with secret tokens, and write each model.
@@ -367,6 +383,13 @@ def prepare_models(self):
             # correct any secret values that point to @@PROP values
             self.fix_property_secrets(variable_map)
 
+            # apply final filter changes for the merged model to the last source model
+            self._apply_final_filters(merged_model_dictionary)
+
+            # check for any content problems in the merged, substituted model
+            content_validator = ContentValidator(self.model_context)
+            content_validator.validate_model(merged_model_dictionary)
+
             target_configuration_helper.generate_all_output_files(Model(merged_model_dictionary), self._aliases,
                                                                   self.credential_injector, self.model_context,
                                                                   ExceptionType.PREPARE)

core/src/main/python/wlsdeploy/tool/util/filter_helper.py

@@ -8,6 +8,7 @@
 
 from wlsdeploy.logging.platform_logger import PlatformLogger
 from wlsdeploy.tool.util.filters import wko_filter
+from wlsdeploy.tool.util.filters import wko_final_filter
 from wlsdeploy.util import dictionary_utils
 from wlsdeploy.util import path_utils
 from wlsdeploy.util.model_translator import FileToPython
@@ -23,7 +24,6 @@
     'k8s_filter': wko_filter.filter_model,
     'vz_filter': wko_filter.filter_model_for_vz,
     'wko_filter': wko_filter.filter_model_for_wko,
-    'wko3_filter': wko_filter.filter_model_for_wko3,
 
     # individual filters for custom target environments
     'online_attributes_filter': wko_filter.filter_online_attributes,
@@ -32,6 +32,15 @@
     'server_ports_filter': wko_filter.check_clustered_server_ports
 }
 
+# final filters run against the merged, substituted model.
+__final_filter_map = {
+    # groups that execute multiple filters
+    'k8s_final_filter': wko_final_filter.filter_final_model,
+    'vz_final_filter': wko_final_filter.filter_final_model_for_vz,
+    'wko_final_filter': wko_final_filter.filter_final_model_for_wko,
+    'wko3_final_filter': wko_final_filter.filter_final_model_for_wko3,
+}
+
 
 def apply_filters(model, tool_type, model_context):
     """
@@ -87,6 +96,39 @@ def apply_filters(model, tool_type, model_context):
     return filter_applied
 
 
+def apply_final_filters(model, update_model, model_context):
+    """
+    Apply any final filters configured to the specified model.
+    :param model: the model to be filtered
+    :param update_model: the model to be updated with any changes
+    :param model_context: used to find target filters
+    :return: True if any filter was applied, False otherwise
+    :raises: BundleAwareException of the specified type: if an error occurs
+    """
+    _method_name = 'apply_final_filters'
+
+    filter_applied = False
+
+    try:
+        final_filters = model_context.get_target_configuration().get_final_model_filters()
+        for final_filter in final_filters:
+            filter_id = dictionary_utils.get_element(final_filter, 'id')
+            filter_method = dictionary_utils.get_element(__final_filter_map, filter_id)
+            if filter_method is None:
+                __logger.severe('WLSDPLY-20037', str_helper.to_string(filter_id), class_name=__class_name,
+                                method_name=_method_name)
+                return False
+            else:
+                filter_method(model, update_model, model_context)
+                return True
+
+    except Exception, ex:
+        __logger.severe('WLSDPLY-20038', str_helper.to_string(ex), error=ex, class_name=__class_name,
+                        method_name=_method_name)
+
+    return filter_applied
+
+
 def _apply_filter(model, the_filter, model_context, filter_file_location):
     """
     Apply the specified filter to the specified model.

core/src/main/python/wlsdeploy/tool/util/filters/wko_filter.py

@@ -7,17 +7,14 @@
 # WDT filters to prepare a model for use a target environment, using the createDomain or prepareModel tools.
 # These operations can be invoked as a single call, or independently of each other.
 from oracle.weblogic.deploy.util import PyRealBoolean
-from oracle.weblogic.deploy.util import PyOrderedDict
 from wlsdeploy.aliases import alias_utils
-from wlsdeploy.aliases.model_constants import ADMIN_SERVER_NAME
 from wlsdeploy.aliases.model_constants import AUTO_MIGRATION_ENABLED
 from wlsdeploy.aliases.model_constants import CALCULATED_LISTEN_PORTS
 from wlsdeploy.aliases.model_constants import CANDIDATE_MACHINE
 from wlsdeploy.aliases.model_constants import CANDIDATE_MACHINES_FOR_MIGRATABLE_SERVER
 from wlsdeploy.aliases.model_constants import CLUSTER
 from wlsdeploy.aliases.model_constants import CLUSTER_MESSAGING_MODE
 from wlsdeploy.aliases.model_constants import DATABASE_LESS_LEASING_BASIS
-from wlsdeploy.aliases.model_constants import DEFAULT_ADMIN_SERVER_NAME
 from wlsdeploy.aliases.model_constants import DYNAMIC_SERVERS
 from wlsdeploy.aliases.model_constants import LISTEN_PORT
 from wlsdeploy.aliases.model_constants import MACHINE
@@ -34,7 +31,6 @@
 from wlsdeploy.aliases.model_constants import RESOURCE_MANAGER
 from wlsdeploy.aliases.model_constants import SECURITY_CONFIGURATION
 from wlsdeploy.aliases.model_constants import SERVER
-from wlsdeploy.aliases.model_constants import SERVER_NAME_PREFIX
 from wlsdeploy.aliases.model_constants import SERVER_START
 from wlsdeploy.aliases.model_constants import SERVER_TEMPLATE
 from wlsdeploy.aliases.model_constants import TOPOLOGY
@@ -49,8 +45,6 @@
 from wlsdeploy.util import dictionary_utils
 import wlsdeploy.util.unicode_helper as str_helper
 
-FIX_PREFIX_TEMPLATE = '-- FIX PREFIX %s --'
-
 _class_name = 'wko_filter'
 _logger = PlatformLogger('wlsdeploy.tool.util')
 
@@ -67,7 +61,6 @@ def filter_model(model, model_context):
     filter_resources(model, model_context)
     filter_online_attributes(model, model_context)
     check_clustered_server_ports(model, model_context)
-    check_dynamic_cluster_prefixes(model, model_context)
 
 
 def filter_model_for_wko(model, model_context):
@@ -80,17 +73,6 @@ def filter_model_for_wko(model, model_context):
     filter_model(model, model_context)
 
 
-def filter_model_for_wko3(model, model_context):
-    """
-    Perform filtering operations on the specified model to prepare for WKO deployment.
-    Currently matches the general k8s target filtering.
-    :param model: the model to be filtered
-    :param model_context: used by nested filters
-    """
-    filter_model(model, model_context)
-    check_admin_server_defined(model, model_context)
-
-
 def filter_model_for_vz(model, model_context):
     """
     Perform filtering operations on the specified model to prepare for Verrazzano deployment.
@@ -161,71 +143,6 @@ def check_clustered_server_ports(model, _model_context):
                 server_port_map[server_cluster] = {"firstServer": server_name, "serverPort": server_port_text}
 
 
-def check_dynamic_cluster_prefixes(model, _model_context):
-    """
-    All Dynamic Clusters must have a DynamicServers section with the ServerNamePrefix field explicitly declared.
-    Ensure each cluster uses a unique value for this field.
-    :param model: the model to be updated
-    :param _model_context: unused, passed by filter_helper if called independently
-    :return:
-    """
-    _method_name = 'check_dynamic_cluster_prefixes'
-
-    server_name_prefixes = []
-    topology_folder = dictionary_utils.get_dictionary_element(model, TOPOLOGY)
-    clusters_folder = dictionary_utils.get_dictionary_element(topology_folder, CLUSTER)
-    for cluster_name, cluster_fields in clusters_folder.items():
-        dynamic_folder = dictionary_utils.get_element(cluster_fields, DYNAMIC_SERVERS)
-        if dynamic_folder:
-            server_name_prefix = dictionary_utils.get_element(dynamic_folder, SERVER_NAME_PREFIX)
-
-            if not server_name_prefix:
-                _logger.warning('WLSDPLY-20204', cluster_name, SERVER_NAME_PREFIX, class_name=_class_name,
-                                method_name=_method_name)
-                server_name_prefix = _get_unused_prefix(server_name_prefixes)
-                dynamic_folder[SERVER_NAME_PREFIX] = server_name_prefix
-
-            elif server_name_prefix in server_name_prefixes:
-                _logger.warning('WLSDPLY-20205', SERVER_NAME_PREFIX, server_name_prefix, class_name=_class_name,
-                                method_name=_method_name)
-                server_name_prefix = _get_unused_prefix(server_name_prefixes)
-                dynamic_folder[SERVER_NAME_PREFIX] = server_name_prefix
-
-            server_name_prefixes.append(server_name_prefix)
-
-
-def check_admin_server_defined(model, _model_context):
-    """
-    Ensure that the AdminServerName attribute is set, and that the server is defined.
-    This is required by WKO 3.0, and not by 4.0 and later.
-    :param model: the model to be filtered
-    :param _model_context: unused, passed by filter_helper if called independently
-    """
-    _method_name = 'check_admin_server_defined'
-
-    topology_folder = dictionary_utils.get_element(model, TOPOLOGY)
-    if topology_folder is None:
-        # for cases with multiple models, avoid adding topology and admin server for
-        # models with only resources, applications, etc.
-        return
-
-    admin_server_name = dictionary_utils.get_element(topology_folder, ADMIN_SERVER_NAME)
-    if not admin_server_name:
-        admin_server_name = DEFAULT_ADMIN_SERVER_NAME
-        _logger.info('WLSDPLY-20206', ADMIN_SERVER_NAME, admin_server_name, class_name=_class_name,
-                     method_name=_method_name)
-        topology_folder[ADMIN_SERVER_NAME] = admin_server_name
-
-    servers_folder = dictionary_utils.get_element(topology_folder, SERVER)
-    if servers_folder is None:
-        servers_folder = PyOrderedDict()
-        topology_folder[SERVER] = servers_folder
-
-    if admin_server_name not in servers_folder:
-        _logger.info('WLSDPLY-20207', SERVER, admin_server_name, class_name=_class_name, method_name=_method_name)
-        servers_folder[admin_server_name] = PyOrderedDict()
-
-
 def filter_topology(model, _model_context):
     """
     Remove elements from the topology section of the model that are not relevant in a Kubernetes environment.
@@ -284,18 +201,6 @@ def filter_resources(model, _model_context):
             del resources[delete_key]
 
 
-def _get_unused_prefix(used_prefixes):
-    """
-    Find a recognizable, unused prefix that can be used in the filtered model.
-    :param used_prefixes: prefixes that have already been used in the model
-    :return: an unused prefix
-    """
-    i = 1
-    while FIX_PREFIX_TEMPLATE % i in used_prefixes:
-        i += 1
-    return FIX_PREFIX_TEMPLATE % i
-
-
 class OnlineAttributeFilter(ModelTraverse):
     """
     Traverse the model and remove any online-only attributes.