JIRA WDT-36 Resolve variables like @@file:@@ORACLE_HOME@@/file.txt@@; always substitute in validate

Author: rakillen

core/src/main/python/create.py

@@ -362,7 +362,7 @@ def main(args):
         variable_map = {}
         if model_context.get_variable_file():
             variable_map = variables.load_variables(model_context.get_variable_file())
-        variables.substitute(model, variable_map)
+        variables.substitute(model, variable_map, model_context)
     except VariableException, ex:
         __logger.severe('WLSDPLY-20004', _program_name, ex.getLocalizedMessage(), error=ex,
                         class_name=_class_name, method_name=_method_name)

core/src/main/python/deploy.py

@@ -448,7 +448,7 @@ def main(args):
         variable_map = {}
         if model_context.get_variable_file():
             variable_map = variables.load_variables(model_context.get_variable_file())
-        variables.substitute(model_dictionary, variable_map)
+        variables.substitute(model_dictionary, variable_map, model_context)
     except VariableException, ex:
         __logger.severe('WLSDPLY-20004', _program_name, ex.getLocalizedMessage(), error=ex,
                         class_name=_class_name, method_name=_method_name)

core/src/main/python/update.py

@@ -466,7 +466,7 @@ def main(args):
         variable_map = {}
         if model_context.get_variable_file():
             variable_map = variables.load_variables(model_context.get_variable_file())
-        variables.substitute(model_dictionary, variable_map)
+        variables.substitute(model_dictionary, variable_map, model_context)
     except VariableException, ex:
         __logger.severe('WLSDPLY-20004', _program_name, ex.getLocalizedMessage(), error=ex,
                         class_name=_class_name, method_name=_method_name)

core/src/main/python/wlsdeploy/tool/validate/validator.py

@@ -222,16 +222,17 @@ def __validate_model_file(self, model_dict, variables_file_name, archive_file_na
         if self._model_file_name is not None:
             self._logger.info('WLSDPLY-05003', self._model_file_name, class_name=_class_name, method_name=_method_name)
 
-        if variables_file_name is not None:
-            self._logger.info('WLSDPLY-05004', variables_file_name, class_name=_class_name, method_name=_method_name)
-            try:
+        try:
+            if variables_file_name is not None:
+                self._logger.info('WLSDPLY-05004', variables_file_name, class_name=_class_name, method_name=_method_name)
                 self._variable_properties = variables.load_variables(variables_file_name)
-                variables.substitute(model_dict, self._variable_properties)
-            except VariableException, ve:
-                ex = exception_helper.create_validate_exception('WLSDPLY-20004', 'validateModel',
-                                                                ve.getLocalizedMessage(), error=ve)
-                self._logger.throwing(ex, class_name=_class_name, method_name=_method_name)
-                raise ex
+
+            variables.substitute(model_dict, self._variable_properties, self._model_context)
+        except VariableException, ve:
+            ex = exception_helper.create_validate_exception('WLSDPLY-20004', 'validateModel',
+                                                            ve.getLocalizedMessage(), error=ve)
+            self._logger.throwing(ex, class_name=_class_name, method_name=_method_name)
+            raise ex
 
         if archive_file_name is not None:
             self._logger.info('WLSDPLY-05005', archive_file_name, class_name=_class_name, method_name=_method_name)

core/src/main/python/wlsdeploy/util/variables.py

@@ -25,6 +25,7 @@
 _variable_pattern = re.compile("\\$\\{[\w.-]+\\}")
 _file_variable_pattern = re.compile("@@FILE:[\w.\\\/:-]+@@")
 _property_pattern = re.compile("@@PROP:[\w.-]+@@")
+_file_nested_variable_pattern = re.compile("@@FILE:@@[\w]+@@[\w.\\\/:-]+@@")
 
 
 def load_variables(file_path):
@@ -126,20 +127,22 @@ def get_variable_names(text):
     return names
 
 
-def substitute(dictionary, variables):
+def substitute(dictionary, variables, model_context):
     """
     Substitute fields in the specified dictionary with variable values.
     :param dictionary: the dictionary in which to substitute variables
     :param variables: a dictionary of variables for substitution
+    :param model_context: used to resolve variables in file paths
     """
-    _process_node(dictionary, variables)
+    _process_node(dictionary, variables, model_context)
 
 
-def _process_node(nodes, variables):
+def _process_node(nodes, variables, model_context):
     """
     Process variables in the node.
     :param nodes: the dictionary to process
     :param variables: the variables to use
+    :param model_context: used to resolve variables in file paths
     """
     # iterate over copy to avoid concurrent change for add/delete
     if type(nodes) is OrderedDict:
@@ -150,22 +153,23 @@ def _process_node(nodes, variables):
         value = nodes[key]
 
         # if the key changes with substitution, remove old key and map value to new key
-        new_key = _substitute(key, variables)
+        new_key = _substitute(key, variables, model_context)
         if new_key is not key:
             nodes.pop(key)
             nodes[new_key] = value
 
         if isinstance(value, dict):
-            _process_node(value, variables)
+            _process_node(value, variables, model_context)
         elif type(value) is str:
-            nodes[key] = _substitute(value, variables)
+            nodes[key] = _substitute(value, variables, model_context)
 
 
-def _substitute(text, variables):
+def _substitute(text, variables, model_context):
     """
     Substitute the variable placeholders with the variable value.
     :param text: the text to process for variable placeholders
     :param variables: the variables to use
+    :param model_context: used to resolve variables in file paths
     :return: the replaced text
     """
     method_name = '_substitute'
@@ -185,6 +189,8 @@ def _substitute(text, variables):
 
     # skip lookups for text with no @@
     if '@@' in text:
+
+        # do properties first, to cover the case @@FILE:/dir/@@PROP:name@@.txt@@
         tokens = _property_pattern.findall(text)
         if tokens:
             for token in tokens:
@@ -204,6 +210,15 @@ def _substitute(text, variables):
                 value = _read_value_from_file(path)
                 text = text.replace(token, value)
 
+        # special case for @@FILE:@@ORACLE_HOME@@/dir/name.txt@@
+        tokens = _file_nested_variable_pattern.findall(text)
+        if tokens:
+            for token in tokens:
+                path = token[7:-2]
+                path = model_context.replace_token_string(path)
+                value = _read_value_from_file(path)
+                text = text.replace(token, value)
+
     return text
 
 

core/src/test/python/variables_test.py

@@ -6,6 +6,7 @@
 
 import wlsdeploy.util.variables as variables
 from oracle.weblogic.deploy.util import VariableException
+from wlsdeploy.util.model_context import ModelContext
 from wlsdeploy.util.model_translator import FileToPython
 
 
@@ -19,14 +20,17 @@ class VariablesTestCase(unittest.TestCase):
     def setUp(self):
         self.name = 'VariablesTestCase'
 
+        # create a context with resource directory as Oracle home, to support @@ORACLE_HOME@@ resolution
+        self.model_context = ModelContext("test", {'-oracle_home': self._resources_dir})
+
     def testReadVariables(self):
         variable_map = variables.load_variables(self._variables_file)
         self.assertEqual(variable_map['my-abc'], 'xyz')
 
     def testSubstituteYaml(self):
         model = FileToPython(self._resources_dir + '/variables-test.yaml', self._use_ordering).parse()
         variable_map = variables.load_variables(self._variables_file)
-        variables.substitute(model, variable_map)
+        variables.substitute(model, variable_map, self.model_context)
         self.assertEqual(model['topology']['Name'], 'xyz123')
         self.assertEqual(model['topology']['Server']['s1']['ListenPort'], '1009')
         self.assertEqual(model['topology']['Server']['s2']['Cluster'], 'myCluster')
@@ -36,7 +40,7 @@ def testSubstituteYaml(self):
     def testSubstituteJson(self):
         model = FileToPython(self._resources_dir + '/variables-test.json', self._use_ordering).parse()
         variable_map = variables.load_variables(self._variables_file)
-        variables.substitute(model, variable_map)
+        variables.substitute(model, variable_map, self.model_context)
         self.assertEqual(model['topology']['Name'], 'xyz123')
         self.assertEqual(model['topology']['Server']['s1']['ListenPort'], '1009')
         self.assertEqual(model['topology']['Server']['s2']['Cluster'], 'myCluster')
@@ -51,7 +55,7 @@ def testVariableNotFound(self):
         model = FileToPython(self._resources_dir + '/variables-test.json', self._use_ordering).parse()
         model['topology']['Name'] = '${bad.variable}'
         variable_map = variables.load_variables(self._variables_file)
-        variables.substitute(model, variable_map)
+        variables.substitute(model, variable_map, self.model_context)
         self.assertEqual(model['topology']['Name'], '${bad.variable}')
 
     def testPropertyNotFound(self):
@@ -62,7 +66,7 @@ def testPropertyNotFound(self):
             model = FileToPython(self._resources_dir + '/variables-test.json', self._use_ordering).parse()
             model['topology']['Name'] = '@@PROP:bad.variable@@'
             variable_map = variables.load_variables(self._variables_file)
-            variables.substitute(model, variable_map)
+            variables.substitute(model, variable_map, self.model_context)
         except VariableException:
             pass
         else:
@@ -71,19 +75,24 @@ def testPropertyNotFound(self):
     def testFileVariable(self):
         path = self._resources_dir + '/' + self._file_variable_name
         model = {'domainInfo': {'AdminUserName': '@@FILE:' + path + '@@'}}
-        variables.substitute(model, {})
+        variables.substitute(model, {}, self.model_context)
         self.assertEqual(model['domainInfo']['AdminUserName'], 'file-variable-value')
 
     def testFileVariableWithVariable(self):
         model = {'domainInfo': {'AdminUserName': '@@FILE:${variable_dir}/' + self._file_variable_name + '@@'}}
-        variables.substitute(model, {'variable_dir': self._resources_dir})
+        variables.substitute(model, {'variable_dir': self._resources_dir}, self.model_context)
+        self.assertEqual(model['domainInfo']['AdminUserName'], 'file-variable-value')
+
+    def testFileVariableWithConstant(self):
+        model = {'domainInfo': {'AdminUserName': '@@FILE:@@ORACLE_HOME@@/' + self._file_variable_name + '@@'}}
+        variables.substitute(model, {'variable_dir': self._resources_dir}, self.model_context)
         self.assertEqual(model['domainInfo']['AdminUserName'], 'file-variable-value')
 
     def testFileVariableNotFound(self):
         try:
             path = self._resources_dir + '/no-file.txt'
             model = {'domainInfo': {'AdminUserName': '@@FILE:' + path + '@@'}}
-            variables.substitute(model, {})
+            variables.substitute(model, {}, self.model_context)
             self.assertEqual(model['domainInfo']['AdminUserName'], 'file-variable-value')
         except VariableException:
             pass