JIRA WDT-475 - Remove unused variables from credential cache (#776)

Author: rakillen

core/src/main/python/discover.py

@@ -417,12 +417,20 @@ def __check_and_customize_model(model, model_context, aliases, credential_inject
 
     credential_cache = None
     if credential_injector is not None:
+        # filter variables or secrets that are no longer in the model
+        credential_injector.filter_unused_credentials(model.get_model())
+
         credential_cache = credential_injector.get_variable_cache()
 
         # Generate k8s create secret script
         if target_configuration.uses_credential_secrets():
             target_configuration_helper.generate_k8s_script(model_context, credential_cache, model.get_model())
 
+        # create additional output after filtering, but before variables have been inserted
+        if model_context.is_targetted_config():
+            target_configuration_helper.create_additional_output(model, model_context, aliases, credential_injector,
+                                                                 ExceptionType.DISCOVER)
+
         # if target handles credential configuration, clear property cache to keep out of variables file.
         if model_context.get_target_configuration().manages_credentials():
             credential_cache.clear()
@@ -510,11 +518,6 @@ def main(args):
     try:
         model = __discover(model_context, aliases, credential_injector, helper)
 
-        if model_context.is_targetted_config():
-            # do this before variables have been inserted into model
-            target_configuration_helper.create_additional_output(model, model_context, aliases, credential_injector,
-                                                                 ExceptionType.DISCOVER)
-
         model = __check_and_customize_model(model, model_context, aliases, credential_injector)
 
     except DiscoverException, ex:

core/src/main/python/prepare_model.py

@@ -239,6 +239,9 @@ def walk(self):
 
         model_file_name = None
 
+        # create a merged model that is not substituted
+        merged_model_dictionary = {}
+
         try:
             model_file_list = self.model_files.split(',')
             for model_file in model_file_list:
@@ -285,6 +288,12 @@ def walk(self):
                 pty._write_dictionary_to_yaml_file(self.current_dict, writer)
                 writer.close()
 
+                cla_helper.merge_model_dictionaries(merged_model_dictionary, self.current_dict, None)
+
+            # filter variables or secrets that are no longer in the merged, filtered model
+            filter_helper.apply_filters(merged_model_dictionary, "discover", self.model_context)
+            self.credential_injector.filter_unused_credentials(merged_model_dictionary)
+
             # use a merged, substituted, filtered model to get domain name and create additional target output.
             full_model_dictionary = cla_helper.load_model(_program_name, self.model_context, self._aliases,
                                                           "discover", WlstModes.OFFLINE)

core/src/main/python/wlsdeploy/tool/util/credential_injector.py

@@ -27,6 +27,7 @@
 from wlsdeploy.util.target_configuration import SECRETS_METHOD
 from wlsdeploy.util.target_configuration_helper import SECRET_PASSWORD_KEY
 from wlsdeploy.util.target_configuration_helper import SECRET_USERNAME_KEY
+from wlsdeploy.util.target_configuration_helper import WEBLOGIC_CREDENTIALS_SECRET_NAME
 
 _class_name = 'CredentialInjector'
 _logger = PlatformLogger('wlsdeploy.tool.util')
@@ -60,6 +61,13 @@ class CredentialInjector(VariableInjector):
         ]
     }
 
+    # keys that should not be filtered from cache, even if they are not in the model.
+    # the model may reference admin credentials indirectly if the target type uses wls_credentials_name.
+    NO_FILTER_KEYS = [
+        WEBLOGIC_CREDENTIALS_SECRET_NAME + ":" + SECRET_PASSWORD_KEY,
+        WEBLOGIC_CREDENTIALS_SECRET_NAME + ":" + SECRET_USERNAME_KEY
+    ]
+
     def __init__(self, program_name, model, model_context, version=None, variable_dictionary=None):
         """
         Construct an instance of the credential injector.
@@ -178,3 +186,50 @@ def get_variable_token(self, attribute, variable_name):
             return target_configuration_helper.format_as_overrides_secret(variable_name)
         else:
             return VariableInjector.get_variable_token(self, attribute, variable_name)
+
+    def filter_unused_credentials(self, model_dictionary):
+        """
+        Remove credentials from the cache that are no longer present in the model.
+        Check for variables or secrets, depending on target configuration.
+        :param model_dictionary: the model to be checked
+        """
+        _method_name = 'filter_unused_credentials'
+
+        target_config = self._model_context.get_target_configuration()
+        credentials_method = target_config.get_credentials_method()
+
+        if credentials_method == CONFIG_OVERRIDES_SECRETS_METHOD:
+            _logger.info("WLSDPLY-19650", credentials_method, class_name=_class_name, method_name=_method_name)
+            return
+
+        all_variables = []
+        self._add_model_variables(model_dictionary, all_variables)
+
+        cache_keys = self.get_variable_cache().keys()
+        for key in cache_keys:
+            if key in self.NO_FILTER_KEYS:
+                continue
+
+            if credentials_method == SECRETS_METHOD:
+                variable_name = '@@SECRET:@@ENV:DOMAIN_UID@@-%s@@' % key
+            else:
+                variable_name = '@@PROP:%s@@' % key
+
+            if variable_name not in all_variables:
+                _logger.info("WLSDPLY-19651", variable_name, class_name=_class_name, method_name=_method_name)
+                del self.get_variable_cache()[key]
+
+    def _add_model_variables(self, model_dictionary, variables):
+        """
+        Add any variable values found in the model dictionary to the variables list
+        :param model_dictionary: the dictionary to be examined
+        :param variables: the list to be appended
+        """
+        for key in model_dictionary:
+            value = model_dictionary[key]
+            if isinstance(value, dict):
+                self._add_model_variables(value, variables)
+            else:
+                text = str(value)
+                if text.startswith('@@'):
+                    variables.append(text)

core/src/main/python/wlsdeploy/util/cla_helper.py

@@ -25,7 +25,7 @@
 from wlsdeploy.util import cla_utils
 from wlsdeploy.util import getcreds
 from wlsdeploy.util import model_helper
-from wlsdeploy.util import model_translator, path_utils
+from wlsdeploy.util import model_translator
 from wlsdeploy.util import path_utils
 
 from wlsdeploy.util import tool_exit
@@ -314,12 +314,12 @@ def merge_model_files(model_file_value, variable_map=None):
 
     for model_file in model_files:
         model = FileToPython(model_file, True).parse()
-        _merge_dictionaries(merged_model, model, variable_map)
+        merge_model_dictionaries(merged_model, model, variable_map)
 
     return merged_model
 
 
-def _merge_dictionaries(dictionary, new_dictionary, variable_map):
+def merge_model_dictionaries(dictionary, new_dictionary, variable_map):
     """
     Merge the values from the new dictionary to the existing one.
     Use variables to resolve keys.
@@ -345,7 +345,7 @@ def _merge_dictionaries(dictionary, new_dictionary, variable_map):
         else:
             value = dictionary[dictionary_key]
             if isinstance(value, dict) and isinstance(new_value, dict):
-                _merge_dictionaries(value, new_value, variable_map)
+                merge_model_dictionaries(value, new_value, variable_map)
             else:
                 dictionary[new_key] = new_value
 

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -1518,6 +1518,10 @@ WLSDPLY-19602=Use variable properties file {0} from command line arguments
 WLSDPLY-19603=Archive file {0} was provided but does not contain a model file
 WLSDPLY-19604=Update model with injected variables
 
+# wlsdeploy/tool/util/credential_injector.py
+WLSDPLY-19650=Unused credential variables will not be filtered for target credentials method {0}
+WLSDPLY-19651=Removing unused credential variable {0}
+
 # wlsdeploy/tool/util/odl_deployer.py
 WLSDPLY-19700=ODL configuration in online mode is not supported, skipping
 WLSDPLY-19701=Unable to add handler {0} without specifying its class at location {1}

core/src/test/python/wlsdeploy/util/cla_helper_test.py

@@ -38,7 +38,7 @@ def testMergeModels(self):
         }
 
         variables = {}
-        cla_helper._merge_dictionaries(dictionary, new_dictionary, variables)
+        cla_helper.merge_model_dictionaries(dictionary, new_dictionary, variables)
         # print("Merged model: " + str(dictionary))
 
         servers = dictionary['Servers']
@@ -59,7 +59,7 @@ def testMergeMatchingProperties(self):
         variables = {}
 
         # no variables are needed to resolve this
-        cla_helper._merge_dictionaries(dictionary, new_dictionary, variables)
+        cla_helper.merge_model_dictionaries(dictionary, new_dictionary, variables)
         # print("Merged model: " + str(dictionary))
 
         servers = dictionary['Servers']
@@ -73,7 +73,7 @@ def testMergeDifferentProperties(self):
         new_dictionary = _build_model_two('@@PROP:server1b@@')
         variables = _build_variable_map()
 
-        cla_helper._merge_dictionaries(dictionary, new_dictionary, variables)
+        cla_helper.merge_model_dictionaries(dictionary, new_dictionary, variables)
         # print("Merged model: " + str(dictionary))
 
         self._check_merged_server(dictionary, '@@PROP:server1a@@')
@@ -84,7 +84,7 @@ def testMergePropertyToName(self):
         new_dictionary = _build_model_two('@@PROP:server1b@@')
         variables = _build_variable_map()
 
-        cla_helper._merge_dictionaries(dictionary, new_dictionary, variables)
+        cla_helper.merge_model_dictionaries(dictionary, new_dictionary, variables)
         # print("Merged model: " + str(dictionary))
 
         self._check_merged_server(dictionary, 'm1')
@@ -95,7 +95,7 @@ def testMergeNameToProperty(self):
         new_dictionary = _build_model_two('m1')
         variables = _build_variable_map()
 
-        cla_helper._merge_dictionaries(dictionary, new_dictionary, variables)
+        cla_helper.merge_model_dictionaries(dictionary, new_dictionary, variables)
         # print("Merged model: " + str(dictionary))
 
         self._check_merged_server(dictionary, '@@PROP:server1a@@')
@@ -106,7 +106,7 @@ def testMergeDeletedNameToName(self):
         new_dictionary = _build_delete_model('m1')
         variables = {}
 
-        cla_helper._merge_dictionaries(dictionary, new_dictionary, variables)
+        cla_helper.merge_model_dictionaries(dictionary, new_dictionary, variables)
         # print("Merged model: " + str(dictionary))
 
         servers = dictionary['Servers']
@@ -118,7 +118,7 @@ def testMergeDeletedNameToDeleteName(self):
         new_dictionary = _build_delete_model('m1')
         variables = {}
 
-        cla_helper._merge_dictionaries(dictionary, new_dictionary, variables)
+        cla_helper.merge_model_dictionaries(dictionary, new_dictionary, variables)
         # print("Merged model: " + str(dictionary))
 
         server = self._check_single_server(dictionary, '!m1')
@@ -130,7 +130,7 @@ def testMergeNameToDeletedName(self):
         new_dictionary = _build_model_two('m1')
         variables = {}
 
-        cla_helper._merge_dictionaries(dictionary, new_dictionary, variables)
+        cla_helper.merge_model_dictionaries(dictionary, new_dictionary, variables)
         # print("Merged model: " + str(dictionary))
 
         server = self._check_single_server(dictionary, 'm1')