Perform checks for compatibility with WKO targets (#1389)

* Check dynamic cluster server name prefixes in target filter

* Ensure admin server is included in model for wko3 targets

* Validate server templates in dynamic clusters

Author: rakillen

core/src/main/python/wlsdeploy/tool/prepare/model_preparer.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, 2022, Oracle and/or its affiliates.
+# Copyright (c) 2021, 2023, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 import os
@@ -163,17 +163,12 @@ def __walk_attribute(self, model_dict, attribute_name, attribute_location):
 
         self._logger.exiting(class_name=_class_name, method_name=_method_name)
 
-    def fix_property_secrets(self):
+    def fix_property_secrets(self, original_variables):
         # Just in case the credential cache has @@PROP in the model's attribute value,
         # we use the original variable file to resolve it,
         # so that the generated json/script files have the resolved property value(s) instead of the @@PROP token.
         # it's possible that the variable file is not specified, or does not exist yet.
 
-        original_variables = {}
-        variable_file = self.model_context.get_variable_file()
-        if variable_file is not None and os.path.exists(variable_file):
-            original_variables = variables.load_variables(variable_file)
-
         credential_caches = self.credential_injector.get_variable_cache()
         for key in credential_caches:
             if variables.is_variable_string(credential_caches[key]):
@@ -359,14 +354,20 @@ def prepare_models(self):
             self._clean_variable_files(merged_model_dictionary)
             self._clean_archive_files()
 
-            # use a merged, substituted, filtered model to get domain name and create additional target output.
-            full_model_dictionary = cla_helper.load_model(_program_name, self.model_context, self._aliases,
-                                                          "discover", WlstModes.OFFLINE)
+            # resolve variables in the model AFTER the clean and filter has been done,
+            # but before generating output files.
+
+            variable_map = {}
+            variable_file = self.model_context.get_variable_file()
+            if variable_file is not None and os.path.exists(variable_file):
+                variable_map = variables.load_variables(variable_file)
+
+            variables.substitute(merged_model_dictionary, variable_map, self.model_context)
 
             # correct any secret values that point to @@PROP values
-            self.fix_property_secrets()
+            self.fix_property_secrets(variable_map)
 
-            target_configuration_helper.generate_all_output_files(Model(full_model_dictionary), self._aliases,
+            target_configuration_helper.generate_all_output_files(Model(merged_model_dictionary), self._aliases,
                                                                   self.credential_injector, self.model_context,
                                                                   ExceptionType.PREPARE)
 

core/src/main/python/wlsdeploy/tool/util/filter_helper.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2022, Oracle and/or its affiliates.
+Copyright (c) 2017, 2023 Oracle and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 import imp
@@ -23,6 +23,7 @@
     'k8s_filter': wko_filter.filter_model,
     'vz_filter': wko_filter.filter_model_for_vz,
     'wko_filter': wko_filter.filter_model_for_wko,
+    'wko3_filter': wko_filter.filter_model_for_wko3,
 
     # individual filters for custom target environments
     'online_attributes_filter': wko_filter.filter_online_attributes,

core/src/main/python/wlsdeploy/tool/util/filters/wko_filter.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021, 2022, Oracle and/or its affiliates.
+# Copyright (c) 2021, 2023, Oracle and/or its affiliates.
 # Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 #
 # ------------
@@ -7,14 +7,17 @@
 # WDT filters to prepare a model for use a target environment, using the createDomain or prepareModel tools.
 # These operations can be invoked as a single call, or independently of each other.
 from oracle.weblogic.deploy.util import PyRealBoolean
+from oracle.weblogic.deploy.util import PyOrderedDict
 from wlsdeploy.aliases import alias_utils
+from wlsdeploy.aliases.model_constants import ADMIN_SERVER_NAME
 from wlsdeploy.aliases.model_constants import AUTO_MIGRATION_ENABLED
 from wlsdeploy.aliases.model_constants import CALCULATED_LISTEN_PORTS
 from wlsdeploy.aliases.model_constants import CANDIDATE_MACHINE
 from wlsdeploy.aliases.model_constants import CANDIDATE_MACHINES_FOR_MIGRATABLE_SERVER
 from wlsdeploy.aliases.model_constants import CLUSTER
 from wlsdeploy.aliases.model_constants import CLUSTER_MESSAGING_MODE
 from wlsdeploy.aliases.model_constants import DATABASE_LESS_LEASING_BASIS
+from wlsdeploy.aliases.model_constants import DEFAULT_ADMIN_SERVER_NAME
 from wlsdeploy.aliases.model_constants import DYNAMIC_SERVERS
 from wlsdeploy.aliases.model_constants import LISTEN_PORT
 from wlsdeploy.aliases.model_constants import MACHINE
@@ -31,6 +34,7 @@
 from wlsdeploy.aliases.model_constants import RESOURCE_MANAGER
 from wlsdeploy.aliases.model_constants import SECURITY_CONFIGURATION
 from wlsdeploy.aliases.model_constants import SERVER
+from wlsdeploy.aliases.model_constants import SERVER_NAME_PREFIX
 from wlsdeploy.aliases.model_constants import SERVER_START
 from wlsdeploy.aliases.model_constants import SERVER_TEMPLATE
 from wlsdeploy.aliases.model_constants import TOPOLOGY
@@ -45,6 +49,8 @@
 from wlsdeploy.util import dictionary_utils
 import wlsdeploy.util.unicode_helper as str_helper
 
+FIX_PREFIX_TEMPLATE = '-- FIX PREFIX %s --'
+
 _class_name = 'wko_filter'
 _logger = PlatformLogger('wlsdeploy.tool.util')
 
@@ -61,6 +67,7 @@ def filter_model(model, model_context):
     filter_resources(model, model_context)
     filter_online_attributes(model, model_context)
     check_clustered_server_ports(model, model_context)
+    check_dynamic_cluster_prefixes(model, model_context)
 
 
 def filter_model_for_wko(model, model_context):
@@ -73,6 +80,17 @@ def filter_model_for_wko(model, model_context):
     filter_model(model, model_context)
 
 
+def filter_model_for_wko3(model, model_context):
+    """
+    Perform filtering operations on the specified model to prepare for WKO deployment.
+    Currently matches the general k8s target filtering.
+    :param model: the model to be filtered
+    :param model_context: used by nested filters
+    """
+    filter_model(model, model_context)
+    check_admin_server_defined(model, model_context)
+
+
 def filter_model_for_vz(model, model_context):
     """
     Perform filtering operations on the specified model to prepare for Verrazzano deployment.
@@ -143,6 +161,71 @@ def check_clustered_server_ports(model, _model_context):
                 server_port_map[server_cluster] = {"firstServer": server_name, "serverPort": server_port_text}
 
 
+def check_dynamic_cluster_prefixes(model, _model_context):
+    """
+    All Dynamic Clusters must have a DynamicServers section with the ServerNamePrefix field explicitly declared.
+    Ensure each cluster uses a unique value for this field.
+    :param model: the model to be updated
+    :param _model_context: unused, passed by filter_helper if called independently
+    :return:
+    """
+    _method_name = 'check_dynamic_cluster_prefixes'
+
+    server_name_prefixes = []
+    topology_folder = dictionary_utils.get_dictionary_element(model, TOPOLOGY)
+    clusters_folder = dictionary_utils.get_dictionary_element(topology_folder, CLUSTER)
+    for cluster_name, cluster_fields in clusters_folder.items():
+        dynamic_folder = dictionary_utils.get_element(cluster_fields, DYNAMIC_SERVERS)
+        if dynamic_folder:
+            server_name_prefix = dictionary_utils.get_element(dynamic_folder, SERVER_NAME_PREFIX)
+
+            if not server_name_prefix:
+                _logger.warning('WLSDPLY-20204', cluster_name, SERVER_NAME_PREFIX, class_name=_class_name,
+                                method_name=_method_name)
+                server_name_prefix = _get_unused_prefix(server_name_prefixes)
+                dynamic_folder[SERVER_NAME_PREFIX] = server_name_prefix
+
+            elif server_name_prefix in server_name_prefixes:
+                _logger.warning('WLSDPLY-20205', SERVER_NAME_PREFIX, server_name_prefix, class_name=_class_name,
+                                method_name=_method_name)
+                server_name_prefix = _get_unused_prefix(server_name_prefixes)
+                dynamic_folder[SERVER_NAME_PREFIX] = server_name_prefix
+
+            server_name_prefixes.append(server_name_prefix)
+
+
+def check_admin_server_defined(model, _model_context):
+    """
+    Ensure that the AdminServerName attribute is set, and that the server is defined.
+    This is required by WKO 3.0, and not by 4.0 and later.
+    :param model: the model to be filtered
+    :param _model_context: unused, passed by filter_helper if called independently
+    """
+    _method_name = 'check_admin_server_defined'
+
+    topology_folder = dictionary_utils.get_element(model, TOPOLOGY)
+    if topology_folder is None:
+        # for cases with multiple models, avoid adding topology and admin server for
+        # models with only resources, applications, etc.
+        return
+
+    admin_server_name = dictionary_utils.get_element(topology_folder, ADMIN_SERVER_NAME)
+    if not admin_server_name:
+        admin_server_name = DEFAULT_ADMIN_SERVER_NAME
+        _logger.info('WLSDPLY-20206', ADMIN_SERVER_NAME, admin_server_name, class_name=_class_name,
+                     method_name=_method_name)
+        topology_folder[ADMIN_SERVER_NAME] = admin_server_name
+
+    servers_folder = dictionary_utils.get_element(topology_folder, SERVER)
+    if servers_folder is None:
+        servers_folder = PyOrderedDict()
+        topology_folder[SERVER] = servers_folder
+
+    if admin_server_name not in servers_folder:
+        _logger.info('WLSDPLY-20207', SERVER, admin_server_name, class_name=_class_name, method_name=_method_name)
+        servers_folder[admin_server_name] = PyOrderedDict()
+
+
 def filter_topology(model, _model_context):
     """
     Remove elements from the topology section of the model that are not relevant in a Kubernetes environment.
@@ -201,6 +284,18 @@ def filter_resources(model, _model_context):
             del resources[delete_key]
 
 
+def _get_unused_prefix(used_prefixes):
+    """
+    Find a recognizable, unused prefix that can be used in the filtered model.
+    :param used_prefixes: prefixes that have already been used in the model
+    :return: an unused prefix
+    """
+    i = 1
+    while FIX_PREFIX_TEMPLATE % i in used_prefixes:
+        i += 1
+    return FIX_PREFIX_TEMPLATE % i
+
+
 class OnlineAttributeFilter(ModelTraverse):
     """
     Traverse the model and remove any online-only attributes.

core/src/main/python/wlsdeploy/tool/validate/content_validator.py

@@ -0,0 +1,63 @@
+"""
+Copyright (c) 2023, Oracle and/or its affiliates.
+Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+"""
+from wlsdeploy.aliases.model_constants import CLUSTER
+from wlsdeploy.aliases.model_constants import DYNAMIC_SERVERS
+from wlsdeploy.aliases.model_constants import SERVER_TEMPLATE
+from wlsdeploy.aliases.model_constants import TOPOLOGY
+from wlsdeploy.logging.platform_logger import PlatformLogger
+from wlsdeploy.util import dictionary_utils
+
+
+class ContentValidator(object):
+    """
+    Class for validating consistency and compatibility of model folders and attributes.
+    These checks are done after alias folder and attribute checks.
+    The model may be a partial model from a multiple-model configuration.
+
+    Dynamic clusters is currently the only validation.
+    Other non-alias validations might fit here (RcuDbInfo?).
+    """
+    _class_name = 'ContentValidator'
+    _logger = PlatformLogger('wlsdeploy.validate')
+
+    def __init__(self, model_context):
+        self._model_context = model_context
+
+    def validate_model(self, model_dict):
+        """
+        Validate the contents of the specified model.
+        :param model_dict: A Python dictionary of the model to be validated
+        :raises ValidationException: if problems occur during validation
+        """
+        _method_name = 'validate_model'
+
+        self.validate_dynamic_clusters(model_dict)
+
+    def validate_dynamic_clusters(self, model_dict):
+        """
+        Validate that dynamic clusters have a unique server template.
+        :param model_dict: A Python dictionary of the model to be validated
+        :raises ValidationException: if problems occur during validation
+        """
+        _method_name = 'validate_dynamic_clusters'
+
+        server_templates = []
+        topology_folder = dictionary_utils.get_dictionary_element(model_dict, TOPOLOGY)
+        clusters_folder = dictionary_utils.get_dictionary_element(topology_folder, CLUSTER)
+        for cluster_name, cluster_fields in clusters_folder.items():
+            dynamic_folder = dictionary_utils.get_element(cluster_fields, DYNAMIC_SERVERS)
+            if dynamic_folder:
+                server_template = dictionary_utils.get_element(dynamic_folder, SERVER_TEMPLATE)
+
+                if not server_template:
+                    self._logger.warning('WLSDPLY-05200', cluster_name, SERVER_TEMPLATE,
+                                         class_name=self._class_name, method_name=_method_name)
+
+                elif server_template in server_templates:
+                    self._logger.warning('WLSDPLY-05201', cluster_name, SERVER_TEMPLATE, server_template,
+                                         class_name=self._class_name, method_name=_method_name)
+
+                else:
+                    server_templates.append(server_template)

core/src/main/python/wlsdeploy/tool/validate/validator.py

@@ -21,6 +21,7 @@
 from wlsdeploy.tool.create import wlsroles_helper
 from wlsdeploy.tool.util.archive_helper import ArchiveHelper
 from wlsdeploy.tool.validate import validation_utils
+from wlsdeploy.tool.validate.content_validator import ContentValidator
 from wlsdeploy.tool.validate.crd_sections_validator import CrdSectionsValidator
 from wlsdeploy.tool.validate.validator_logger import ValidatorLogger
 from wlsdeploy.util import dictionary_utils
@@ -265,6 +266,9 @@ def __validate_model_file(self, model_dict, variables_map, archive_file_name):
             k8s_validator = CrdSectionsValidator(self._model_context)
             k8s_validator.validate_model(model_dict)
 
+        content_validator = ContentValidator(self._model_context)
+        content_validator.validate_model(model_dict)
+
         self._logger.exiting(class_name=_class_name, method_name=_method_name)
 
     def load_variables(self, variables_file_name):

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -549,6 +549,10 @@ WLSDPLY-05043=Model location {0} does not match any of the {1} folder options
 # wlsdeploy/tool/validate/crd_sections_validator.py
 WLSDPLY-05090=Model folder {0} is not supported, will be skipped
 
+# wlsdeploy/tool/validate/content_validator.py
+WLSDPLY-05200=Dynamic cluster "{0}" does not have a {1} value, which will prevent the domain from starting
+WLSDPLY-05201=Dynamic cluster "{0}" {1} "{2}" is used by another cluster, which is not allowed
+
 # wlsdeploy/tools/validate/validation_utils.py
 WLSDPLY-05300=NOT USED
 
@@ -1771,6 +1775,10 @@ WLSDPLY-20036={0} encountered an unexpected runtime exception.  Stacktrace: {1}
 WLSDPLY-20201=Unsupported attribute {0} at location {1} removed from model
 WLSDPLY-20202=Changing {0} to false for {1} "{2}"
 WLSDPLY-20203={0} entries "{1}" and "{2}" in {3} "{4}" have different {5} values: {6} and {7}
+WLSDPLY-20204=Dynamic cluster "{0}" does not specify {1}, which will cause deployment to fail
+WLSDPLY-20205=The {0} value "{1}" is used by multiple dynamic clusters, which will cause deployment to fail
+WLSDPLY-20206=Adding {0} value "{1}" to model for WKO 3 compatibility
+WLSDPLY-20207=Adding {0} "{1}" to model for WKO 3 compatibility
 
 # Common messages used for tool exit and clean-up
 WLSDPLY-21000={0} Messages:

core/src/main/targetconfigs/wko-dii/target.json

@@ -1,7 +1,7 @@
 {
   "model_filters" : {
         "discover": [
-          { "id": "wko_filter" }
+          { "id": "wko3_filter" }
         ]
   },
   "variable_injectors" : {"PORT": {},"HOST": {},"URL": {}},

core/src/main/targetconfigs/wko-pv/target.json

@@ -1,7 +1,7 @@
 {
   "model_filters" : {
         "discover": [
-          { "id": "wko_filter" }
+          { "id": "wko3_filter" }
         ]
   },
   "variable_injectors" : {"PORT": {},"HOST": {},"URL": {}},

core/src/main/targetconfigs/wko/target.json

@@ -2,7 +2,7 @@
   "model_filters" : {
         "discover": [
           { "name": "wko_prep", "path": "@@TARGET_CONFIG_DIR@@/wko_operator_filter.py" },
-          { "id": "wko_filter" }
+          { "id": "wko3_filter" }
         ]
   },
   "variable_injectors" : {"PORT": {},"HOST": {},"URL": {}},