Merge pull request #1053 from oracle/WDT-539-bypass-domainBin-for-targets

Wdt 539 provide configurable setting for targets to exclude domain bin scripts in discover and prepare model

Author: robertpatrick

core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2021, Oracle Corporation and/or its affiliates.
+ * Copyright (c) 2017, 2022, Oracle Corporation and/or its affiliates.
  * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
  */
 package oracle.weblogic.deploy.util;
@@ -854,6 +854,15 @@ public void extractDomainBinScript(String archivePath, File extractToLocation) t
         extractFileFromZip(archivePath, ARCHIVE_DOM_BIN_TARGET_DIR, "", extractToLocation);
         LOGGER.exiting(CLASS, METHOD);
     }
+
+    public void removeDomainBinScripts() throws WLSDeployArchiveIOException {
+        final String METHOD = "removeDomainBinScripts";
+
+        LOGGER.entering(CLASS, METHOD);
+        getZipFile().removeZipEntries(ARCHIVE_DOM_BIN_TARGET_DIR + ZIP_SEP);
+        LOGGER.exiting(CLASS, METHOD);
+    }
+
     /**
      * This method adds a classpath library to the archive.  If a library with the same name already exists, this
      * method assumes that the new one also needs to be added so it changes the name to prevent conflicts by adding

core/src/main/python/prepare_model.py

@@ -36,7 +36,8 @@
 ]
 
 __optional_arguments = [
-    CommandLineArgUtil.VARIABLE_FILE_SWITCH
+    CommandLineArgUtil.VARIABLE_FILE_SWITCH,
+    CommandLineArgUtil.ARCHIVE_FILE_SWITCH
 ]
 
 
@@ -55,8 +56,9 @@ def __process_args(args):
     target_configuration_helper.process_target_arguments(argument_map)
 
     model_context = ModelContext(_program_name, argument_map)
-    # override this check, since archive file is not supplied here
-    model_context.get_validate_configuration().set_allow_unresolved_archive_references(True)
+    if model_context.get_archive_file_name() is None:
+        # override this check, if archive file is not supplied
+        model_context.get_validate_configuration().set_allow_unresolved_archive_references(True)
     return model_context
 
 

core/src/main/python/wlsdeploy/tool/discover/domain_info_discoverer.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2021, Oracle Corporation and/or its affiliates.
+Copyright (c) 2017, 2022, Oracle Corporation and/or its affiliates.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 import glob
@@ -109,31 +109,35 @@ def get_domain_libs(self):
     def get_user_env_scripts(self):
         """
         Look for the user overrides scripts run in setDomainEnv in the domain bin directory
+        If discover is running with the -target <type> do not collect the domainBin scripts
         :raise: DiscoverException: an unexpected exception occurred writing a jar file to the archive file
         """
         _method_name = 'get_user_env_scripts'
         _logger.entering(class_name=_class_name, method_name=_method_name)
-        archive_file = self._model_context.get_archive_file()
-        domain_bin = self._convert_path('bin')
         entries = []
-        if os.path.isdir(domain_bin):
-            search_directory = FileUtils.fixupFileSeparatorsForJython(os.path.join(domain_bin, "setUserOverrides*.*"))
-            _logger.finer('WLSDPLY-06425', search_directory, class_name=_class_name, method_name=_method_name)
-            file_list = glob.glob(search_directory)
-            if file_list:
-                _logger.finer('WLSDPLY-06423', domain_bin, class_name=_class_name, method_name=_method_name)
-                for entry in file_list:
-                    try:
-                        updated_name = archive_file.addDomainBinScript(File(entry))
-                    except WLSDeployArchiveIOException, wioe:
-                        de = exception_helper.create_discover_exception('WLSDPLY-06426', entry,
-                                                                        wioe.getLocalizedMessage())
-                        _logger.throwing(class_name=_class_name, method_name=_method_name, error=de)
-                        raise de
-
-                    entries.append(updated_name)
-                    _logger.finer('WLSDPLY-06424', entry, updated_name, class_name=_class_name,
-                                  method_name=_method_name)
+        if self._model_context.get_target_configuration().exclude_domain_bin_contents():
+            _logger.fine('WLSDPLY-06427', class_name=_class_name, method_name=_method_name)
+        else:
+            archive_file = self._model_context.get_archive_file()
+            domain_bin = self._convert_path('bin')
+            if os.path.isdir(domain_bin):
+                search_directory = FileUtils.fixupFileSeparatorsForJython(os.path.join(domain_bin, "setUserOverrides*.*"))
+                _logger.finer('WLSDPLY-06425', search_directory, class_name=_class_name, method_name=_method_name)
+                file_list = glob.glob(search_directory)
+                if file_list:
+                    _logger.finer('WLSDPLY-06423', domain_bin, class_name=_class_name, method_name=_method_name)
+                    for entry in file_list:
+                        try:
+                            updated_name = archive_file.addDomainBinScript(File(entry))
+                        except WLSDeployArchiveIOException, wioe:
+                            de = exception_helper.create_discover_exception('WLSDPLY-06426', entry,
+                                                                            wioe.getLocalizedMessage())
+                            _logger.throwing(class_name=_class_name, method_name=_method_name, error=de)
+                            raise de
+
+                        entries.append(updated_name)
+                        _logger.finer('WLSDPLY-06424', entry, updated_name, class_name=_class_name,
+                                      method_name=_method_name)
 
         _logger.exiting(class_name=_class_name, method_name=_method_name, result=entries)
         return model_constants.DOMAIN_SCRIPTS, entries

core/src/main/python/wlsdeploy/tool/prepare/model_preparer.py

@@ -13,11 +13,13 @@
 from wlsdeploy.aliases.aliases import Aliases
 from wlsdeploy.aliases.location_context import LocationContext
 from wlsdeploy.aliases.model_constants import DOMAIN_INFO
+from wlsdeploy.aliases.model_constants import DOMAIN_SCRIPTS
 from wlsdeploy.aliases.wlst_modes import WlstModes
 from wlsdeploy.exception import exception_helper
 from wlsdeploy.exception.expection_types import ExceptionType
 from wlsdeploy.logging.platform_logger import PlatformLogger
 from wlsdeploy.tool.util import filter_helper
+from wlsdeploy.tool.util.archive_helper import ArchiveHelper
 from wlsdeploy.tool.util.credential_injector import CredentialInjector
 from wlsdeploy.tool.util.variable_injector import VARIABLE_FILE_UPDATE
 from wlsdeploy.tool.util.variable_injector import VariableInjector
@@ -221,6 +223,10 @@ def _apply_filter_and_inject_variable(self, model_dict, model_context):
         else:
             credential_properties = self.credential_injector.get_variable_cache()
 
+        if target_config.exclude_domain_bin_contents() and DOMAIN_INFO in model_dict \
+                and DOMAIN_SCRIPTS in model_dict[DOMAIN_INFO]:
+            del model_dict[DOMAIN_INFO][DOMAIN_SCRIPTS]
+
         variable_injector = VariableInjector(_program_name, model_dict, model_context,
                                              WebLogicHelper(self._logger).get_actual_weblogic_version(),
                                              credential_properties)
@@ -279,6 +285,23 @@ def _clean_variable_files(self, merged_model_dictionary):
                     self._logger.warning('WLSDPLY-05803', e.getLocalizedMessage(),
                                          class_name=_class_name, method_name=_method_name)
 
+    def _clean_archive_files(self):
+        """
+        Remove any content necessary from the archive file.
+        """
+        _method_name = '_clean_archive_files'
+
+        archive_file_name = self.model_context.get_archive_file_name()
+        if archive_file_name is None:
+            return
+
+        # If the archive file(s) exist, always make a copy even if we aren't filtering them
+        archive_helper = ArchiveHelper(archive_file_name, None, self._logger, exception_helper.ExceptionType.PREPARE)
+        archive_helper = archive_helper.copy_archives_to_target_directory(self.model_context.get_output_dir())
+
+        if self.model_context.get_target_configuration().exclude_domain_bin_contents():
+            archive_helper.remove_domain_scripts()
+
     def prepare_models(self):
         """
         Replace password attributes in each model file with secret tokens, and write each model.
@@ -327,9 +350,10 @@ def prepare_models(self):
                 if variable_file is not None and not os.path.exists(variable_file):
                     variable_file = None
 
+                archive_file_name = self.model_context.get_archive_file_name()
                 return_code = validator.validate_in_tool_mode(model_dictionary,
                                                               variables_file_name=variable_file,
-                                                              archive_file_name=None)
+                                                              archive_file_name=archive_file_name)
 
                 if return_code == Validator.ReturnCode.STOP:
                     ex = exception_helper.create_prepare_exception('WLSDPLY-05804', model_file_name)
@@ -359,6 +383,7 @@ def prepare_models(self):
             filter_helper.apply_filters(merged_model_dictionary, "discover", self.model_context)
             self.credential_injector.filter_unused_credentials(merged_model_dictionary)
             self._clean_variable_files(merged_model_dictionary)
+            self._clean_archive_files()
 
             # use a merged, substituted, filtered model to get domain name and create additional target output.
             full_model_dictionary = cla_helper.load_model(_program_name, self.model_context, self._aliases,

core/src/main/python/wlsdeploy/tool/util/archive_helper.py

@@ -1,7 +1,10 @@
 """
-Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
+Copyright (c) 2017, 2022, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
+import os
+import shutil
+
 from java.io import File
 from java.lang import IllegalArgumentException
 from java.lang import IllegalStateException
@@ -359,6 +362,25 @@ def extract_domain_bin_script(self, script_path):
         self.__logger.exiting(class_name=self.__class_name, method_name=_method_name)
         return
 
+    def remove_domain_scripts(self):
+        """
+        Remove any domain bin scripts from the archive.
+        :raises: BundleAwareException of the appropriate type: if an error occurs
+        """
+        _method_name = 'remove_domain_scripts'
+
+        self.__logger.entering(class_name=self.__class_name, method_name=_method_name)
+        for archive_file in self.__archive_files:
+            try:
+                archive_file.removeDomainBinScripts()
+            except (WLSDeployArchiveIOException, IllegalArgumentException), e:
+                ex = exception_helper.create_exception(self.__exception_type, 'WLSDPLY-19311',
+                                                       archive_file.getArchiveFileName(), e.getLocalizedMessage(),
+                                                       error=e)
+                self.__logger.throwing(ex, class_name=self.__class_name, method_name=_method_name)
+                raise ex
+        self.__logger.exiting(class_name=self.__class_name, method_name=_method_name)
+
     def get_archive_entries(self):
         """
         Get the entries from all the archives.
@@ -439,6 +461,46 @@ def get_manifest(self, source_path):
             return archive.getManifest(source_path)
         return None
 
+    def copy_archives_to_target_directory(self, target_directory):
+        """
+        Makes a copy of the specified archive file(s) to the target directory.
+        :param target_directory: the directory to which the archive file(s) will be copied
+        :return: new ArchiveHelper for the new archive file(s)
+        :raises: BundleAwareException of the appropriate type: if an error occurred while copying the file
+        """
+        _method_name = 'copy_archives_to_target_directory'
+        self.__logger.entering(target_directory, class_name=self.__class_name, method_name=_method_name)
+
+        file_names = self.__archive_files_text.split(CommandLineArgUtil.ARCHIVE_FILES_SEPARATOR)
+        new_file_names = []
+        for file_name in file_names:
+            new_file_name = self._copy_file(file_name, target_directory)
+            new_file_names.append(new_file_name)
+        new_archive_file_text = CommandLineArgUtil.ARCHIVE_FILES_SEPARATOR.join(new_file_names)
+        return ArchiveHelper(new_archive_file_text, self.__domain_home, self.__logger, self.__exception_type)
+
+    def _copy_file(self, source_file_name, target_directory):
+        """
+        Make a copy of the source file to the target directory
+        :param source_file_name: source file name
+        :param target_directory: target directory name
+        :return: file name of the new file in the target directory
+        :raises: BundleAwareException of the appropriate type: if an error occurred while copying the file
+        """
+        _method_name = '_copy_file'
+
+        self.__logger.entering(source_file_name, target_directory, class_name=self.__class_name, method_name=_method_name)
+        base_name = os.path.basename(source_file_name)
+        target_file_name = os.path.join(target_directory, base_name)
+        try:
+            shutil.copyfile(source_file_name, target_file_name)
+        except Exception, e:
+            ex = exception_helper.create_exception(self.__exception_type, 'WLSDPLY-19312', source_file_name,
+                                                   target_file_name, e.getLocalizedMessage(), error=e)
+            self.__logger.throwing(ex, class_name=self.__class_name, method_name=_method_name)
+            raise ex
+        return target_file_name
+
     def _find_archive_for_path(self, path, required=False):
         """
         Find the archive file containing the specified path.

core/src/main/python/wlsdeploy/util/cla_utils.py

@@ -18,6 +18,7 @@
 import oracle.weblogic.deploy.util.FileUtils as JFileUtils
 
 from wlsdeploy.exception import exception_helper
+from wlsdeploy.json.json_translator import JsonToPython
 from wlsdeploy.logging.platform_logger import PlatformLogger
 from wlsdeploy.util import path_utils
 from wlsdeploy.util import validate_configuration
@@ -1157,8 +1158,9 @@ def _validate_target_arg(self, value):
         else:
             try:
                 # verify the file is in proper format
-                file_handle = open(target_configuration_file)
-                config_dictionary = eval(file_handle.read())
+                # file_handle = open(target_configuration_file)
+                # config_dictionary = eval(file_handle.read())
+                config_dictionary = JsonToPython(target_configuration_file).parse()
                 target_configuration = TargetConfiguration(config_dictionary)
                 validation_method = target_configuration.get_validation_method()
                 if (validation_method is not None) and \

core/src/main/python/wlsdeploy/util/model_context.py

@@ -13,6 +13,7 @@
 from oracle.weblogic.deploy.util import XPathUtil
 from wlsdeploy.aliases.model_constants import MODEL_LIST_DELIMITER
 from wlsdeploy.aliases.wlst_modes import WlstModes
+from wlsdeploy.json.json_translator import JsonToPython
 from wlsdeploy.logging import platform_logger
 from wlsdeploy.util import validate_configuration
 from wlsdeploy.util.cla_utils import CommandLineArgUtil
@@ -644,8 +645,7 @@ def get_target_configuration(self):
             if self._target:
                 target_configuration_file = self.get_target_configuration_file()
                 if os.path.exists(target_configuration_file):
-                    file_handle = open(target_configuration_file)
-                    configuration_dict = eval(file_handle.read())
+                    configuration_dict = JsonToPython(target_configuration_file).parse()
 
             self._target_configuration = TargetConfiguration(configuration_dict)
 

core/src/main/python/wlsdeploy/util/target_configuration.py

@@ -15,6 +15,9 @@
 # Overrides the Kubernetes secret name for the WebLogic admin user credential
 WLS_CREDENTIALS_NAME = "wls_credentials_name"
 
+# Determines whether the domainBin contents should be excluded
+EXCLUDE_DOMAIN_BIN_CONTENTS = "exclude_domain_bin_contents"
+
 # put secret tokens in the model, and build a script to create the secrets.
 SECRETS_METHOD = 'secrets'
 
@@ -134,3 +137,15 @@ def manages_credentials(self):
         :return: True if credential values are managed, False otherwise
         """
         return self.get_credentials_method() in [SECRETS_METHOD, CONFIG_OVERRIDES_SECRETS_METHOD]
+
+    def exclude_domain_bin_contents(self):
+        """
+        Determine if the contents of the domain's bin directory should be
+        excluded from the model and archive.  If True, these files will be
+        excluded and not go into the model or archive file.
+        :return: True if the domain bin contents should be excluded, False otherwise
+        """
+        result = dictionary_utils.get_element(self.config_dictionary, EXCLUDE_DOMAIN_BIN_CONTENTS)
+        if result is None:
+            result = False
+        return result

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -705,6 +705,7 @@ WLSDPLY-06423=Add the user override env script files from the domain bin locatio
 WLSDPLY-06424=Add user override script {0} to the archive file and entry {1] to the model domain info section
 WLSDPLY-06425=Look for user env override scripts using search pattern {0}
 WLSDPLY-06426=Unexpected exception occurred writing the domain bin user overrides file {0} to the archive : {1}
+WLSDPLY-06427=Will not collect domainBin user override scripts for -target command line options.
 
 # global_resources_discoverer.py
 WLSDPLY-06440=Discover Global Resources from the domain
@@ -1520,6 +1521,8 @@ WLSDPLY-19307=Unable to extract classpath libraries from archive file {0} to dom
 WLSDPLY-19308=Failed to extract script to domain bin {0} because it does not exist in archive file {1}
 WLSDPLY-19309=Unable to extract from domain bin {0} from archive file {1}: {2}
 WLSDPLY-19310=Unable to extract user custom files from archive file {0} to domain directory {1}: {2}
+WLSDPLY-19311=Unable to remove domain bin scripts from archive file{0}: {1}
+WLSDPLY-19312=Unable to copy file {0} to {1}: {2}
 
 # wlsdeploy/tool/util/topology_helper.py
 WLSDPLY-19400=Creating placeholder for server template {0}

core/src/main/targetconfigs/vz/target.json

@@ -9,6 +9,7 @@
     "validation_method" : "lax",
     "credentials_method" : "secrets",
     "credentials_output_method" : "script",
+    "exclude_domain_bin_contents": true,
     "wls_credentials_name" : "__weblogic-credentials__",
     "additional_secrets": "runtime-encryption-secret",
     "additional_output" : "vz-application.yaml"