Add Kubernetes configuration to model (#521)

* Initial commit, Window and Unix scripts and entry python module

* Support kubernetes section in model

* Cleaned up comments

* Corrected usage, fix windows exit code issue

* Get child folders and attribute locations for model sections; update print-usage

* Provide default for environment variable lookup

* Return copy of list that caller can change

* Use revised methods to get folder names and attribute location

* Use revised methods to get folder names and attribute location; validate kubernetes section

* Revised message for multiple tools

* Create the domain resource file with content from the model

* Disable kubernetes section validation temporarily

* Re-enable kubernetes section validation; update alias file; write known values to resource file

* Revised aliases to match domain resource format

* Process named folders in model

* Add option to write list of dictionaries

* Revised aliases, added ServerPod

* Use dictionary lists in domain resource file

* Allow contained alias files for all model folders

* Update domain resource file with command-line domain home, and cluster info from the model

* Added defaults for several domain resource values

* Use different validation of domain home for extract tool

* Add WebLogic credentials secret if not present

* Updated copyrights

* Revised comment

* Remove usage comments related to domain type

* Added default for spec/image

* Check image pull policy before adding image pull secrets

Author: rakillen

core/src/main/python/create.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 
 The main module for the WLSDeploy tool to create empty domains.
@@ -48,6 +48,7 @@
 from wlsdeploy.util import wlst_extended
 from wlsdeploy.util import wlst_helper
 from wlsdeploy.util.cla_utils import CommandLineArgUtil
+from wlsdeploy.util.cla_utils import TOOL_TYPE_CREATE
 from wlsdeploy.util.weblogic_helper import WebLogicHelper
 from wlsdeploy.tool.create import atp_helper
 
@@ -92,7 +93,7 @@ def __process_args(args):
     """
     cla_util = CommandLineArgUtil(_program_name, __required_arguments, __optional_arguments)
     cla_util.set_allow_multiple_models(True)
-    required_arg_map, optional_arg_map = cla_util.process_args(args, True)
+    required_arg_map, optional_arg_map = cla_util.process_args(args, TOOL_TYPE_CREATE)
     __verify_required_args_present(required_arg_map)
     __process_java_home_arg(optional_arg_map)
     __process_domain_location_args(optional_arg_map)

core/src/main/python/extract_resource.py

@@ -0,0 +1,145 @@
+"""
+Copyright (c) 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
+Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
+
+The entry point for the extractDomainResource tool.
+"""
+import os
+import sys
+
+from oracle.weblogic.deploy.deploy import DeployException
+from oracle.weblogic.deploy.util import CLAException
+from oracle.weblogic.deploy.util import WebLogicDeployToolingVersion
+
+sys.path.append(os.path.dirname(os.path.realpath(sys.argv[0])))
+
+# imports from local packages start here
+from wlsdeploy.aliases.aliases import Aliases
+from wlsdeploy.aliases.wlst_modes import WlstModes
+from wlsdeploy.exception.expection_types import ExceptionType
+from wlsdeploy.logging.platform_logger import PlatformLogger
+from wlsdeploy.tool.extract.domain_resource_extractor import DomainResourceExtractor
+from wlsdeploy.tool.util import model_context_helper
+from wlsdeploy.tool.util.wlst_helper import WlstHelper
+from wlsdeploy.util import cla_helper
+from wlsdeploy.util import tool_exit
+from wlsdeploy.util import wlst_extended
+from wlsdeploy.util.cla_utils import CommandLineArgUtil
+from wlsdeploy.util.cla_utils import TOOL_TYPE_EXTRACT
+from wlsdeploy.util.model import Model
+from wlsdeploy.util.weblogic_helper import WebLogicHelper
+
+wlst_extended.wlst_functions = globals()
+
+_program_name = 'extractDomainResource'
+_class_name = 'extract_resource'
+__logger = PlatformLogger('wlsdeploy.extract')
+__wls_helper = WebLogicHelper(__logger)
+__wlst_helper = WlstHelper(__logger, ExceptionType.DEPLOY)
+__wlst_mode = WlstModes.OFFLINE
+
+__required_arguments = [
+    CommandLineArgUtil.ORACLE_HOME_SWITCH,
+    CommandLineArgUtil.DOMAIN_HOME_SWITCH,
+    CommandLineArgUtil.DOMAIN_RESOURCE_FILE_SWITCH
+]
+
+__optional_arguments = [
+    # Used by shell script to locate WLST
+    CommandLineArgUtil.DOMAIN_TYPE_SWITCH,
+    CommandLineArgUtil.ARCHIVE_FILE_SWITCH,
+    CommandLineArgUtil.MODEL_FILE_SWITCH,
+    CommandLineArgUtil.VARIABLE_FILE_SWITCH,
+    CommandLineArgUtil.USE_ENCRYPTION_SWITCH,
+    CommandLineArgUtil.PASSPHRASE_SWITCH,
+]
+
+
+def __process_args(args):
+    """
+    Process the command-line arguments and prompt the user for any missing information
+    :param args: the command-line arguments list
+    :raises CLAException: if an error occurs while validating and processing the command-line arguments
+    """
+    cla_util = CommandLineArgUtil(_program_name, __required_arguments, __optional_arguments)
+    cla_util.set_allow_multiple_models(True)
+    required_arg_map, optional_arg_map = cla_util.process_args(args, TOOL_TYPE_EXTRACT)
+
+    cla_helper.verify_required_args_present(_program_name, __required_arguments, required_arg_map)
+    cla_helper.validate_optional_archive(_program_name, optional_arg_map)
+
+    # determine if the model file was passed separately or requires extraction from the archive.
+    cla_helper.validate_model_present(_program_name, optional_arg_map)
+    cla_helper.validate_variable_file_exists(_program_name, optional_arg_map)
+    cla_helper.process_encryption_args(optional_arg_map)
+
+    combined_arg_map = optional_arg_map.copy()
+    combined_arg_map.update(required_arg_map)
+    return model_context_helper.create_context(_program_name, combined_arg_map)
+
+
+def __extract_resource(model, model_context, aliases):
+    """
+    Offline deployment orchestration
+    :param model: the model
+    :param model_context: the model context
+    :param aliases: the aliases object
+    :raises: DeployException: if an error occurs
+    """
+    _method_name = '__extract_resource'
+
+    resource_extractor = DomainResourceExtractor(model, model_context, aliases, __logger)
+    resource_extractor.extract()
+    return 0
+
+
+def main(args):
+    """
+    The python entry point for extractDomainResource.
+    :param args: the command-line arguments
+    """
+    _method_name = 'main'
+
+    __logger.entering(args[0], class_name=_class_name, method_name=_method_name)
+    for index, arg in enumerate(args):
+        __logger.finer('sys.argv[{0}] = {1}', str(index), str(arg), class_name=_class_name, method_name=_method_name)
+
+    __wlst_helper.silence()
+
+    exit_code = CommandLineArgUtil.PROG_OK_EXIT_CODE
+
+    try:
+        model_context = __process_args(args)
+    except CLAException, ex:
+        exit_code = ex.getExitCode()
+        if exit_code != CommandLineArgUtil.HELP_EXIT_CODE:
+            __logger.severe('WLSDPLY-20008', _program_name, ex.getLocalizedMessage(), error=ex,
+                            class_name=_class_name, method_name=_method_name)
+        cla_helper.clean_up_temp_files()
+
+        # create a minimal model for summary logging
+        model_context = model_context_helper.create_exit_context(_program_name)
+        tool_exit.end(model_context, exit_code)
+
+    aliases = Aliases(model_context, wlst_mode=__wlst_mode)
+
+    model_dictionary = cla_helper.load_model(_program_name, model_context, aliases, "extract", __wlst_mode)
+
+    try:
+        model = Model(model_dictionary)
+        exit_code = __extract_resource(model, model_context, aliases)
+    except DeployException, ex:
+        __logger.severe('WLSDPLY-09015', _program_name, ex.getLocalizedMessage(), error=ex,
+                        class_name=_class_name, method_name=_method_name)
+        cla_helper.clean_up_temp_files()
+        tool_exit.end(model_context, CommandLineArgUtil.PROG_ERROR_EXIT_CODE)
+
+    cla_helper.clean_up_temp_files()
+
+    tool_exit.end(model_context, exit_code)
+    return
+
+
+if __name__ == '__main__' or __name__ == 'main':
+    WebLogicDeployToolingVersion.logVersionInfo(_program_name)
+    main(sys.argv)

core/src/main/python/wlsdeploy/aliases/alias_entries.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 import copy
@@ -45,12 +45,19 @@
 from wlsdeploy.aliases.alias_constants import WLST_SUBFOLDERS_PATH
 from wlsdeploy.aliases.alias_constants import WLST_TYPE
 from wlsdeploy.aliases.location_context import LocationContext
+from wlsdeploy.aliases.model_constants import APP_DEPLOYMENTS
 from wlsdeploy.aliases.model_constants import APPLICATION
-from wlsdeploy.aliases.model_constants import RCU_DB_INFO
-from wlsdeploy.aliases.model_constants import WLS_ROLES
-from wlsdeploy.aliases.model_constants import ODL_CONFIGURATION
+from wlsdeploy.aliases.model_constants import DOMAIN_INFO
 from wlsdeploy.aliases.model_constants import DOMAIN_INFO_ALIAS
+from wlsdeploy.aliases.model_constants import KUBERNETES_ALIAS
+from wlsdeploy.aliases.model_constants import ODL_CONFIGURATION
+from wlsdeploy.aliases.model_constants import KUBERNETES
+from wlsdeploy.aliases.model_constants import RCU_DB_INFO
 from wlsdeploy.aliases.model_constants import RESOURCE_MANAGER
+from wlsdeploy.aliases.model_constants import RESOURCES
+from wlsdeploy.aliases.model_constants import SERVER_POD
+from wlsdeploy.aliases.model_constants import TOPOLOGY
+from wlsdeploy.aliases.model_constants import WLS_ROLES
 from wlsdeploy.aliases.validation_codes import ValidationCodes
 from wlsdeploy.aliases.wlst_modes import WlstModes
 from wlsdeploy.exception import exception_helper
@@ -137,6 +144,31 @@ class AliasEntries(object):
         WLS_ROLES
     ]
 
+    __kubernetes_top_level_folders = [
+        'metadata',
+        'spec'
+    ]
+
+    __section_top_folders_map = {
+        DOMAIN_INFO: __domain_info_top_level_folders,
+        TOPOLOGY: __topology_top_level_folders,
+        RESOURCES: __resources_top_level_folders,
+        APP_DEPLOYMENTS: __app_deployments_top_level_folders,
+        KUBERNETES: __kubernetes_top_level_folders
+    }
+
+    # in rare cases, the alias file name does not match the folder name
+    __alternate_alias_file_name_map = {
+        APPLICATION: 'AppDeployment',
+        'metadata': 'Metadata',
+        'spec': 'Spec',
+        'serverPod': 'ServerPod'
+    }
+
+    # all the categories that appear at the top of model sections
+    __top_model_categories = []
+
+    # all the categories, including section-attribute and contained categories
     __all_model_categories = []
 
     __domain_name_token = 'DOMAIN'
@@ -156,16 +188,23 @@ def __init__(self, wlst_mode=WlstModes.OFFLINE, wls_version=None):
             self._wls_helper = WebLogicHelper(_logger, wls_version)
             self._wls_version = wls_version
 
-        self.__all_model_categories.extend(self.__topology_top_level_folders)
-        self.__all_model_categories.extend(self.__resources_top_level_folders)
-        self.__all_model_categories.extend(self.__app_deployments_top_level_folders)
-        self.__all_model_categories.extend(self.__domain_info_top_level_folders)
+        # top model categories
+        self.__top_model_categories.extend(self.__topology_top_level_folders)
+        self.__top_model_categories.extend(self.__resources_top_level_folders)
+        self.__top_model_categories.extend(self.__app_deployments_top_level_folders)
+        self.__top_model_categories.extend(self.__domain_info_top_level_folders)
+        self.__top_model_categories.extend(self.__kubernetes_top_level_folders)
 
-        # ResourceManager is not a top-level folder, it's contained by ResourceManagement
+        # all model categories
+        self.__all_model_categories.extend(self.__top_model_categories)
+
+        # include contained categories
         self.__all_model_categories.append(RESOURCE_MANAGER)
+        self.__all_model_categories.append(SERVER_POD)
 
-        # DomainInfo is not a top-level folder, it defines domainInfo top-level attributes
+        # include section attribute categories
         self.__all_model_categories.append(DOMAIN_INFO_ALIAS)
+        self.__all_model_categories.append(KUBERNETES_ALIAS)
         return
 
     def get_dictionary_for_location(self, location, resolve=True):
@@ -196,15 +235,7 @@ def get_model_domain_subfolder_names(self):
         _method_name = 'get_model_domain_subfolder_names'
 
         _logger.entering(class_name=_class_name, method_name=_method_name)
-        folder_list = list(self.__all_model_categories)
-        #
-        # Remove all folders that do not appear at the WLST root level
-        #
-        if 'Domain' in folder_list:
-            folder_list.remove('Domain')
-        if 'ResourceManager' in folder_list:
-            folder_list.remove('ResourceManager')
-
+        folder_list = list(self.__top_model_categories)
         _logger.exiting(class_name=_class_name, method_name=_method_name, result=folder_list)
         return folder_list
 
@@ -244,12 +275,31 @@ def get_model_app_deployments_subfolder_names(self):
         """
         return list(self.__app_deployments_top_level_folders)
 
-    def get_model_domain_info_subfolder_names(self):
+    def get_model_section_subfolder_names(self, section_name):
         """
-        Get the top-level model folder names underneath the domain info section.
+        Get the top-level model folder names underneath the specified section.
         :return: a list of the folder names
         """
-        return list(self.__domain_info_top_level_folders)
+        result = dictionary_utils.get_element(self.__section_top_folders_map, section_name)
+        if result is None:
+            result = []
+        return result
+
+    def get_model_section_attribute_location(self, section_name):
+        """
+        Get the location containing the attributes for a model section (topology, domainInfo, etc.)
+        :return: a location, or None of the section does not have attributes.
+        """
+        if section_name == TOPOLOGY:
+            return LocationContext()
+
+        if section_name == DOMAIN_INFO:
+            return LocationContext().append_location(DOMAIN_INFO_ALIAS)
+
+        if section_name == KUBERNETES:
+            return LocationContext().append_location(KUBERNETES_ALIAS)
+
+        return None
 
     def get_model_subfolder_names_for_location(self, location):
         """
@@ -883,13 +933,13 @@ def _unit_test_only_get_category_map_files(self):
     def _get_category_file_prefix(self, category_name):
         """
         Return the file prefix for the specified top-level category.
-        The file prefix should match the category name exactly.
-        File name for Application is different for some reason
+        The file prefix should match the category name exactly, except in rare cases.
         :param category_name: the name to be checked
-        :return: the corressponding file name prefix
+        :return: the corresponding file name prefix
         """
-        if category_name == APPLICATION:
-            return 'AppDeployment'
+        alternate_name = dictionary_utils.get_element(self.__alternate_alias_file_name_map, category_name)
+        if alternate_name is not None:
+            return alternate_name
         return category_name
 
     def __get_dictionary_for_location(self, location, resolve_path_tokens=True):
@@ -1041,9 +1091,9 @@ def __load_contains_categories(self, model_category_name, raw_model_dict, base_p
                 for key, value in raw_folders_wlst_paths.iteritems():
                     raw_model_dict[WLST_PATHS][key] = base_path + value
 
-            for folder in raw_model_dict_folders:
-                raw_folder_dict = raw_model_dict_folders[folder]
-                self.__load_contains_categories(folder, raw_folder_dict, base_path)
+        for folder in raw_model_dict_folders:
+            raw_folder_dict = raw_model_dict_folders[folder]
+            self.__load_contains_categories(folder, raw_folder_dict, base_path)
 
         #
         # Now that the folder paths are all updated accordingly, load any contains folders, compute the

core/src/main/python/wlsdeploy/aliases/aliases.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 from java.lang import String
@@ -126,13 +126,20 @@ def get_model_app_deployments_top_level_folder_names(self):
         """
         return self._alias_entries.get_model_app_deployments_subfolder_names()
 
-    def get_model_domain_info_top_level_folder_names(self):
+    def get_model_section_top_level_folder_names(self, section_name):
         """
-        Returns a list of the recognized top-level model folders in the domainInfo section corresponding to the
+        Returns a list of the recognized top-level model folders in the specified section corresponding to the
         known WLST top-level folders.
         :return: a list of the recognized top-level model folder names
         """
-        return self._alias_entries.get_model_domain_info_subfolder_names()
+        return self._alias_entries.get_model_section_subfolder_names(section_name)
+
+    def get_model_section_attribute_location(self, section_name):
+        """
+        Get the location containing the attributes for a model section (topology, domainInfo, etc.)
+        :return: a location, or None of the section does not have attributes.
+        """
+        return self._alias_entries.get_model_section_attribute_location(section_name)
 
     def get_model_subfolder_names(self, location):
         """