Jira WDT-385 Enable exploded app in archive (#512)

* JIRA-WDT-385 Unzip exploded app correctly

* JIRA-WDT-385 Read app manifest from file if needed; fix online deployment

* JIRA-WDT-385 Use manifest methods to update shared library name

Author: rakillen

core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+ * Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
  * Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
  */
 package oracle.weblogic.deploy.util;
@@ -379,6 +379,24 @@ public String extractFile(String path, File extractToLocation) throws WLSDeployA
         return extractFile(path, extractToLocation, false);
     }
 
+    /**
+     * Extract the specified directory to the specified location (which is typically the domain home).  For example,
+     * if the path is wlsdeploy/applications/myapp and the extractToLocation is the domain home, the directory
+     * will be written to $DOMAIN_HOME/wlsdeploy/applications/myapp .
+     *
+     * @param path              the path into the archive file to extract
+     * @param extractToLocation the base directory to which to write the extracted directory.
+     * @return the canonical extracted directory name
+     * @throws WLSDeployArchiveIOException if an error occurs reading the archive or writing the directory
+     * @throws IllegalArgumentException    if the path is null or empty or the extractToLocation
+     *                                     was not a valid, existing directory
+     */
+    public String extractDirectory(String path, File extractToLocation) throws WLSDeployArchiveIOException {
+        String result = FileUtils.getCanonicalFile(new File(extractToLocation, path)).getAbsolutePath();
+        this.extractDirectoryFromZip(path, extractToLocation);
+        return result;
+    }
+
     /**
      * Extract the specified file to the specified location.
      *

core/src/main/python/wlsdeploy/tool/deploy/applications_deployer.py

@@ -1,16 +1,18 @@
 """
-Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 import copy
 import os
 from java.io import ByteArrayOutputStream
 from java.io import File
+from java.io import FileInputStream
 from java.io import FileNotFoundException
 from java.io import IOException
 from java.lang import IllegalStateException
 from java.security import NoSuchAlgorithmException
 from java.util.jar import JarFile
+from java.util.jar import Manifest
 from java.util.zip import ZipException
 from sets import Set
 from wlsdeploy.aliases.location_context import LocationContext
@@ -164,7 +166,7 @@ def __add_applications(self):
 
             if deployer_utils.is_path_into_archive(app_source_path):
                 if self.archive_helper is not None:
-                    self.archive_helper.extract_file(app_source_path)
+                    self.__extract_source_path_from_archive(app_source_path, APPLICATION, application_name)
                 else:
                     ex = exception_helper.create_deploy_exception('WLSDPLY-09303', application_name)
                     self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name)
@@ -567,14 +569,23 @@ def __build_app_deploy_strategy(self, location, model_apps, existing_apps, exist
                         self.model_context.replace_tokens(APPLICATION, app, param, app_dict)
 
                 if app in existing_apps:
+                    # Compare the hashes of the domain's existing apps to the model's apps.
+                    # If they match, remove them from the list to be deployed.
+                    # If they are different, stop and un-deploy the app, and leave it in the list.
+
                     existing_app_ref = dictionary_utils.get_dictionary_element(existing_app_refs, app)
                     plan_path = dictionary_utils.get_element(existing_app_ref, 'planPath')
                     src_path = dictionary_utils.get_element(existing_app_ref, 'sourcePath')
 
-                    model_src_hash = \
-                        self.__get_hash(dictionary_utils.get_element(app_dict, SOURCE_PATH))
-                    model_plan_hash = \
-                        self.__get_hash(dictionary_utils.get_element(app_dict, PLAN_PATH))
+                    model_src_path = dictionary_utils.get_element(app_dict, SOURCE_PATH)
+
+                    if (model_src_path is not None) and deployer_utils.is_path_into_archive(model_src_path) \
+                            and self.archive_helper.contains_path(model_src_path):
+                        model_src_hash = -1  # don't calculate hash, just ensure that hashes will not match
+                    else:
+                        model_src_hash = self.__get_hash(model_src_path)
+
+                    model_plan_hash = self.__get_hash(dictionary_utils.get_element(app_dict, PLAN_PATH))
 
                     existing_src_hash = self.__get_file_hash(src_path)
                     existing_plan_hash = self.__get_file_hash(plan_path)
@@ -603,6 +614,10 @@ def __get_file_hash(self, filename):
         try:
             if filename is None:
                 return None
+
+            if File(filename).isDirectory():  # can't calculate for exploded apps, libraries, etc.
+                return None
+
             hash_value = FileUtils.computeHash(filename)
         except (IOException, NoSuchAlgorithmException), e:
             ex = exception_helper.create_deploy_exception('WLSDPLY-09309', filename, e.getLocalizedMessage(), error=e)
@@ -734,7 +749,8 @@ def __deploy_model_applications(self, model_apps, app_location, deployed_applist
                 options = _get_deploy_options(model_apps, app_name, library_module='false')
                 for uses_path_tokens_attribute_name in uses_path_tokens_attribute_names:
                     if uses_path_tokens_attribute_name in app_dict:
-                        self.__extract_file_from_archive(app_dict[uses_path_tokens_attribute_name])
+                        self.__extract_source_path_from_archive(app_dict[uses_path_tokens_attribute_name],
+                                                                APPLICATION, app_name)
 
                 location.add_name_token(token_name, app_name)
                 resource_group_template_name, resource_group_name, partition_name = \
@@ -823,6 +839,31 @@ def __extract_file_from_archive(self, path):
             self.archive_helper.extract_file(path)
         return
 
+    def __extract_source_path_from_archive(self, source_path, model_type, model_name):
+        """
+        Extract contents from the archive set for the specified source path.
+        The contents may be a single file, or a directory with exploded content.
+        :param source_path: the path to be extracted (previously checked to be under wlsdeploy)
+        :param model_type: the model type (Application, etc.), used for logging
+        :param model_name: the element name (my-app, etc.), used for logging
+        """
+        _method_name = '__extract_source_path_from_archive'
+
+        # source path may be may be a single file (jar, war, etc.)
+        if self.archive_helper.contains_file(source_path):
+            self.archive_helper.extract_file(source_path)
+
+        # source path may be exploded directory in archive
+        elif self.archive_helper.contains_path(source_path):
+            self.archive_helper.extract_directory(source_path)
+
+        else:
+            ex = exception_helper.create_deploy_exception('WLSDPLY-09330', model_type, model_name, source_path)
+            self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name)
+            raise ex
+
+        return
+
     def __get_deployable_library_versioned_name(self, source_path, model_name):
         """
         Get the proper name of the deployable library that WLST requires in the target domain.  This method is
@@ -840,49 +881,28 @@ def __get_deployable_library_versioned_name(self, source_path, model_name):
 
         old_name_tuple = deployer_utils.get_library_name_components(model_name, self.wlst_mode)
         try:
+            versioned_name = old_name_tuple[self._EXTENSION_INDEX]
             source_path = self.model_context.replace_token_string(source_path)
-            archive = JarFile(source_path)
-            manifest_object = archive.getManifest()
-            tokens = []
-            if manifest_object is not None:
-                bao = ByteArrayOutputStream()
-                manifest_object.write(bao)
-                manifest = bao.toString('UTF-8')
-                tokens = manifest.split()
-
-            if 'Extension-Name:' in tokens:
-                extension_index = tokens.index('Extension-Name:')
-                if len(tokens) > extension_index:
-                    versioned_name = tokens[extension_index + 1]
-                else:
-                    ex = exception_helper.create_deploy_exception('WLSDPLY-09321', model_name, source_path, tokens)
-                    self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name)
-                    raise ex
-            else:
-                versioned_name = old_name_tuple[self._EXTENSION_INDEX]
+            manifest = self.__get_manifest(source_path)
+            if manifest is not None:
+                attributes = manifest.getMainAttributes()
+
+                extension_name = attributes.getValue("Extension-Name")
+                if not string_utils.is_empty(extension_name):
+                    versioned_name = extension_name
 
-            if 'Specification-Version:' in tokens:
-                spec_index = tokens.index('Specification-Version:')
-                if len(tokens) > spec_index:
-                    versioned_name += '#' + tokens[spec_index + 1]
+                specification_version = attributes.getValue("Specification-Version")
+                if not string_utils.is_empty(specification_version):
+                    versioned_name += '#' + specification_version
 
                     # Cannot specify an impl version without a spec version
-                    if 'Implementation-Version:' in tokens:
-                        impl_index = tokens.index('Implementation-Version:')
-                        if len(tokens) > impl_index:
-                            versioned_name += '@' + tokens[impl_index + 1]
-                        else:
-                            ex = exception_helper.create_deploy_exception('WLSDPLY-09322', model_name,
-                                                                          source_path, tokens)
-                            self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name)
-                            raise ex
-                else:
-                    ex = exception_helper.create_deploy_exception('WLSDPLY-09323', model_name, source_path, tokens)
-                    self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name)
-                    raise ex
+                    implementation_version = attributes.getValue("Implementation-Version")
+                    if not string_utils.is_empty(implementation_version):
+                        versioned_name += '@' + implementation_version
+
+                self.logger.info('WLSDPLY-09324', model_name, versioned_name,
+                                 class_name=self._class_name, method_name=_method_name)
 
-            self.logger.info('WLSDPLY-09324', model_name, versioned_name,
-                             class_name=self._class_name, method_name=_method_name)
         except (IOException, FileNotFoundException, ZipException, IllegalStateException), e:
             ex = exception_helper.create_deploy_exception('WLSDPLY-09325', model_name, source_path, str(e), error=e)
             self.logger.throwing(ex, class_name=self._class_name, method_name=_method_name)
@@ -912,8 +932,8 @@ def __get_deployable_application_versioned_name(self, source_path, model_name):
 
         try:
             source_path = self.model_context.replace_token_string(source_path)
-            archive = JarFile(source_path)
-            manifest = archive.getManifest()
+            manifest = self.__get_manifest(source_path)
+
             if manifest is not None:
                 attributes = manifest.getMainAttributes()
                 application_version = attributes.getValue(self._APP_VERSION_MANIFEST_KEY)
@@ -930,6 +950,37 @@ def __get_deployable_application_versioned_name(self, source_path, model_name):
         self.logger.exiting(class_name=self._class_name, method_name=_method_name, result=versioned_name)
         return versioned_name
 
+    def __get_manifest(self, source_path):
+        """
+        Returns the manifest object for the specified path.
+        The source path may be a jar, or an exploded path.
+        :param source_path: the source path to be checked
+        :return: the manifest, or None if it is not present
+        :raises: IOException: if there are problems reading an existing manifest
+        """
+        source_path_file = File(source_path)
+        manifest = None
+
+        if source_path_file.isDirectory():
+            manifest_file = File(source_path_file, "META-INF/MANIFEST.MF")
+            if manifest_file.exists():
+                stream = None
+                try:
+                    stream = FileInputStream(manifest_file)
+                    manifest = Manifest(stream)
+                finally:
+                    if stream is not None:
+                        try:
+                            stream.close()
+                        except IOException:
+                            # nothing to report
+                            pass
+        else:
+            archive = JarFile(source_path)
+            manifest = archive.getManifest()
+
+        return manifest
+
     def __get_deployment_ordering(self, apps):
         _method_name = '__get_deployment_ordering'
         name_sorted_keys = apps.keys()

core/src/main/python/wlsdeploy/tool/deploy/deployer.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 import os
@@ -385,6 +385,9 @@ def __process_archive_entry(self, location, key, value):
                         # The file does not exist so extract it from the archive.
                         self.archive_helper.extract_file(value)
                         result = True
+        elif self.archive_helper.contains_path(value):
+            # contents should have been extracted elsewhere, such as for apps and shared libraries
+            result = self.__process_directory_entry(fullpath)
         elif value.endswith('/'):
             # If the path is a directory in the wlsdeploy directory tree
             # but not in the archive, just create it to help the user.

core/src/main/python/wlsdeploy/tool/util/archive_helper.py

@@ -1,5 +1,5 @@
 """
-Copyright (c) 2017, 2019, Oracle Corporation and/or its affiliates.  All rights reserved.
+Copyright (c) 2017, 2020, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
 from java.io import File
@@ -206,6 +206,32 @@ def extract_file(self, path, location=None):
         self.__logger.exiting(class_name=self.__class_name, method_name=_method_name, result=result)
         return result
 
+    def extract_directory(self, path, location=None):
+        """
+        Extract the specified directory from the archive into the specified directory, or into Domain Home.
+        :param path: the path into the archive
+        :param location: the location to which to extract the directory
+        :return: path to the extracted directory
+        :raises: BundleAwareException of the appropriate type: if an error occurs
+        """
+        _method_name = 'extract_directory'
+        self.__logger.entering(path, class_name=self.__class_name, method_name=_method_name)
+
+        try:
+            archive_file = self._find_archive_for_path(path, True)
+            if location is None:
+                result = archive_file.extractDirectory(path, self.__domain_home)
+            else:
+                extract_location = FileUtils.getCanonicalFile(File(location))
+                result = archive_file.extractDirectory(path, extract_location, True)
+        except (IllegalArgumentException, WLSDeployArchiveIOException), e:
+            ex = exception_helper.create_exception(self.__exception_type, "WLSDPLY-19303", path,
+                                                   self.__archive_files_text, e.getLocalizedMessage(), error=e)
+            self.__logger.throwing(ex, class_name=self.__class_name, method_name=_method_name)
+            raise ex
+        self.__logger.exiting(class_name=self.__class_name, method_name=_method_name, result=result)
+        return result
+
     def get_file_hash(self, path):
         """
         Get the Base64-encoded hash value for the file at the specified path within the archive.

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -991,7 +991,7 @@ WLSDPLY-09326=Deployment order is {0}
 WLSDPLY-09327=Failed to stop application {0}: {1}
 WLSDPLY-09328=Application {0} name has been updated to {1} to match the application version in the MANIFEST.MF file
 WLSDPLY-09329=Failed to compute name for application {0} from archive at {1}: {2}
-
+WLSDPLY-09330=Source path for {0} {1} not found in any archive: {2}
 
 # wlsdeploy/tool/deploy/common_resources_deployer.py
 WLSDPLY-09400=ResourceGroup was specified in the test file but are not supported in WebLogic Server version {0}