Collect jdbc wallet (#1379)

* add wallet collection logic

* undo unnecessary changes

* refactor

* fix bug

* fix bug

* add integration test

* make sure wallet available when updating ds

* restore integration tests

* Fix npe

* add error when wallet file not found

* copyright change

* add info for not trying to collect if remote

* fix remote discovery and skip archive

* fix problem where domain directory does not exist yet before extracting wallet

* undo model change

* fix compare test result

* fix compare result

* fix sonar issue

* sonar fix

* sonar code smell

Author: jshum2479

core/src/main/java/oracle/weblogic/deploy/util/WLSDeployArchive.java

@@ -1394,7 +1394,7 @@ public int removeStructuredApplication(String appPath, boolean silent) throws WL
      * @param shlibPath file name to find the name for
      * @return name for model archive file name
      */
-    public String getSharedLibraryArchivePath(String shlibPath) {
+    public static String getSharedLibraryArchivePath(String shlibPath) {
         return getArchiveName(ARCHIVE_SHLIBS_TARGET_DIR, shlibPath);
     }
 
@@ -1718,7 +1718,7 @@ public int removeSharedLibraryDeploymentPlan(String planPath, boolean silent) th
      * @param domainLibPath the file name to get the archive file name
      * @return model ready archive file name
      */
-    public String getDomainLibArchiveName(String domainLibPath) {
+    public static String getDomainLibArchiveName(String domainLibPath) {
         return getArchiveName(ARCHIVE_DOMLIB_TARGET_DIR, domainLibPath);
     }
 
@@ -3718,6 +3718,16 @@ public String addDatabaseWallet(String walletName, String sourceLocation) throws
         return newName;
     }
 
+    /**
+     * Return the database wallet path of a file.
+     * @param walletName database wallet name
+     * @param fileLocation a file in the wallet
+     * @return path to the particular file in the database wallet
+     */
+    public static String getDatabaseWalletArchivePath(String walletName, String fileLocation) {
+        return getArchiveName(ARCHIVE_DB_WALLETS_DIR + ZIP_SEP + walletName, fileLocation);
+    }
+
     /**
      * Add the RCU database wallet to the archive.
      *

core/src/main/python/create.py

@@ -320,10 +320,16 @@ def main(model_context):
             archive_helper = ArchiveHelper(archive_file_name, domain_path, __logger, ExceptionType.CREATE)
 
         has_atp, has_ssl = validate_rcu_args_and_model(model_context, model_dictionary, archive_helper, aliases)
-        # check if there is an atpwallet and extract in the domain dir
-        # it is to support non JRF domain but user wants to use ATP database
-        if has_atp and archive_helper:
-            archive_helper.extract_database_wallet()
+
+        if archive_helper:
+            domain_parent = model_context.get_domain_parent_dir()
+            domain_home = model_context.get_domain_home()
+            if domain_parent and domain_home is None:
+                domain_home = os.path.join(domain_parent, model_dictionary[model_constants.TOPOLOGY]['Name'])
+            if not os.path.exists(os.path.abspath(domain_home)):
+                os.mkdir(os.path.abspath(domain_home))
+
+            archive_helper.extract_all_database_wallets()
 
         creator = DomainCreator(model_dictionary, model_context, aliases)
         creator.create()

core/src/main/python/wlsdeploy/tool/deploy/datasource_deployer.py

@@ -18,6 +18,8 @@ class DatasourceDeployer(Deployer):
 
     def __init__(self, model, model_context, aliases, wlst_mode=WlstModes.OFFLINE):
         Deployer.__init__(self, model, model_context, aliases, wlst_mode)
+        if not model_context.is_remote() and self.archive_helper:
+            self.archive_helper.extract_all_database_wallets()
 
     def add_data_sources(self, parent_dict, location):
         """

core/src/main/python/wlsdeploy/tool/discover/common_resources_discoverer.py

@@ -2,6 +2,7 @@
 Copyright (c) 2017, 2023, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
+import os.path
 from java.lang import IllegalArgumentException
 
 from oracle.weblogic.deploy.util import PyOrderedDict as OrderedDict
@@ -91,6 +92,7 @@ def get_datasources(self):
         location = LocationContext(self._base_location)
         location.append_location(model_top_folder_name)
         datasources = self._find_names_in_folder(location)
+        collected_wallet = {}
         if datasources is not None:
             _logger.info('WLSDPLY-06340', len(datasources), class_name=_class_name, method_name=_method_name)
             typedef = self._model_context.get_domain_typedef()
@@ -112,11 +114,126 @@ def get_datasources(self):
                         resource_result = result[datasource][model_second_folder]
                         self._populate_model_parameters(resource_result, location)
                         self._discover_subfolders(resource_result, location)
+                        self._collect_jdbc_driver_wallet(datasource, collected_wallet,
+                                  result[datasource][model_constants.JDBC_RESOURCE][model_constants.JDBC_DRIVER_PARAMS])
                         location.remove_name_token(name_token)
                         location.pop_location()
         _logger.exiting(class_name=_class_name, method_name=_method_name, result=result)
         return model_top_folder_name, result
 
+    def _collect_jdbc_driver_wallet(self, datasource, collected_wallet, driver_params):
+        if not self._model_context.skip_archive() and model_constants.JDBC_DRIVER_PARAMS_PROPERTIES in driver_params:
+            properties = driver_params[model_constants.JDBC_DRIVER_PARAMS_PROPERTIES]
+            self._update_wallet_property_and_collect_files(collected_wallet, datasource, properties)
+
+    def _update_wallet_property_and_collect_files(self, collected_wallet, datasource, properties):
+        _method_name = '_update_wallet_property_and_collect_files'
+        for connection_property in [model_constants.DRIVER_PARAMS_KEYSTORE_PROPERTY,
+                                    model_constants.DRIVER_PARAMS_TRUSTSTORE_PROPERTY,
+                                    model_constants.DRIVER_PARAMS_NET_TNS_ADMIN]:
+            if connection_property in properties:
+                qualified_property_value = properties[connection_property]['Value']
+                if qualified_property_value:
+                    if qualified_property_value.startswith(WLSDeployArchive.WLSDPLY_ARCHIVE_BINARY_DIR
+                                                           + WLSDeployArchive.ZIP_SEP):
+                        qualified_property_value = os.path.join(self._model_context.get_domain_home()
+                                                                , qualified_property_value)
+                    if os.path.exists(qualified_property_value):
+                        fixed_path = self._add_wallet_directory_to_archive(datasource, collected_wallet,
+                                                                           qualified_property_value)
+                        _logger.info('WLSDPLY-06367', connection_property, fixed_path,
+                                     class_name=_class_name, method_name=_method_name)
+                        properties[connection_property]['Value'] = fixed_path
+                    else:
+                        _logger.severe('WLSDPLY-06370', datasource, connection_property,
+                                       properties[connection_property]['Value'])
+                        de = exception_helper.create_discover_exception('WLSDPLY-06370', datasource,
+                                                                        connection_property,
+                                                                        properties[connection_property][
+                                                                            'Value'])
+                        _logger.throwing(class_name=_class_name, method_name=_method_name, error=de)
+                        raise de
+
+    def _add_wallet_directory_to_archive(self, datasource, collected_wallet_dictionary, property_value):
+        # error if wallet parent path is at oracle home
+        onprem_wallet_parent_path, wallet_name = self._get_wallet_name_and_path(datasource, property_value)
+
+        if self._model_context.is_remote():
+            fixed_path = WLSDeployArchive.getDatabaseWalletArchivePath(wallet_name, property_value)
+            self.add_to_remote_map(property_value, fixed_path,
+                                   WLSDeployArchive.ArchiveEntryType.DB_WALLET.name())
+            return fixed_path
+
+        onprem_wallet_dir_is_not_flat = self._is_on_prem_wallet_dir_flat(onprem_wallet_parent_path, property_value)
+        archive_file = self._model_context.get_archive_file()
+
+        # keep track of the wallet name and from where it is collected
+        #  { <onprem_wallet_parentpath> : { 'wallet_name': <name>, 'path_into_archive': <wallet path into archive>,
+        #      },
+        #     .... }
+        #
+        if onprem_wallet_parent_path not in collected_wallet_dictionary:
+            if onprem_wallet_dir_is_not_flat:
+                # collect the specific file
+                fixed_path = archive_file.addDatabaseWallet(wallet_name, os.path.abspath(property_value))
+                path_into_archive = os.path.dirname(fixed_path)
+            else:
+                fixed_path = archive_file.addDatabaseWallet(wallet_name, onprem_wallet_parent_path)
+                path_into_archive = fixed_path
+                fixed_path = os.path.join(fixed_path, os.path.basename(property_value))
+
+            collected_wallet_dictionary[onprem_wallet_parent_path] = \
+                {'wallet_name' : wallet_name, 'path_into_archive': path_into_archive}
+        else:
+            # if the directory has already been visited before
+            # check for the wallet to see if the file has already been collected on prem and add only the file
+            check_path = os.path.join(
+                collected_wallet_dictionary[onprem_wallet_parent_path]['path_into_archive'],
+                os.path.basename(property_value))
+            if check_path not in archive_file.getArchiveEntries() and os.path.isfile(property_value):
+                # only case is it is not flat directory if the particular file has not been collected before, add
+                # it to the previous wallet
+                fixed_path = archive_file.addDatabaseWallet(collected_wallet_dictionary[onprem_wallet_parent_path]['wallet_name'],
+                                                        os.path.abspath(property_value))
+            else:
+                # already in archive just figure out the path
+                if (os.path.isdir(property_value)):
+                    fixed_path = collected_wallet_dictionary[onprem_wallet_parent_path]['path_into_archive']
+                else:
+                    fixed_path = os.path.join(collected_wallet_dictionary[onprem_wallet_parent_path]['path_into_archive'],
+                                              os.path.basename(property_value))
+
+        return fixed_path
+
+    def _get_wallet_name_and_path(self, datasource, property_value):
+        _method_name = '_get_wallet_name_and_path'
+        # fix up name just in case
+        wallet_name = datasource.replace(' ', '').replace('(', '').replace(')', '')
+        if os.path.isdir(property_value):
+            onprem_wallet_parent_path = os.path.abspath(property_value)
+        else:
+            onprem_wallet_parent_path = os.path.dirname(os.path.abspath(property_value))
+        if self._model_context.get_oracle_home() == onprem_wallet_parent_path:
+            _logger.severe('WLSDPLY-06368', property_value, self._model_context.get_oracle_home())
+            de = exception_helper.create_discover_exception('WLSDPLY-06368', property_value,
+                                                            self._model_context.get_oracle_home())
+            _logger.throwing(class_name=_class_name, method_name=_method_name, error=de)
+            raise de
+        return onprem_wallet_parent_path, wallet_name
+
+    def _is_on_prem_wallet_dir_flat(self, onprem_wallet_parent_path, property_value):
+        # info if wallet parent is not flat and only collect the particular file and not the entire directory
+        dir_list = os.listdir(onprem_wallet_parent_path)
+        onprem_wallet_dir_is_not_flat = False
+        for item in dir_list:
+            if os.path.isdir(os.path.join(onprem_wallet_parent_path, item)):
+                onprem_wallet_dir_is_not_flat = True
+                break
+        # put out info for user only particular file is collected and not the entire wallet directory
+        if onprem_wallet_dir_is_not_flat:
+            _logger.info('WLSDPLY-06369', property_value)
+        return onprem_wallet_dir_is_not_flat
+
     def get_foreign_jndi_providers(self):
         """
         Discover Foreign JNDI providers from the domain.
@@ -224,7 +341,6 @@ def archive_file_store_directory(self, file_store_name, file_store_dictionary):
                         file_store_dictionary[model_constants.DIRECTORY] = new_source_name
 
         _logger.exiting(class_name=_class_name, method_name=_method_name)
-        return
 
     def get_jdbc_stores(self):
         """

core/src/main/python/wlsdeploy/tool/util/archive_helper.py

@@ -2,7 +2,7 @@
 Copyright (c) 2017, 2023, Oracle Corporation and/or its affiliates.  All rights reserved.
 Licensed under the Universal Permissive License v 1.0 as shown at https://oss.oracle.com/licenses/upl.
 """
-import os
+import os, sets
 import shutil
 
 from java.io import File
@@ -401,6 +401,21 @@ def extract_database_wallet(self, wallet_name=WLSDeployArchive.DEFAULT_RCU_WALLE
         self.__logger.exiting(class_name=self.__class_name, method_name=_method_name, result=resulting_wallet_path)
         return resulting_wallet_path
 
+    def extract_all_database_wallets(self):
+
+        for archive_file in self.__archive_files:
+            archive_entries = archive_file.getArchiveEntries()
+            wallet_names = sets.Set()
+            for entry in archive_entries:
+                if entry.startswith(WLSDeployArchive.ARCHIVE_DB_WALLETS_DIR):
+                    if os.path.isdir(entry):
+                        wallet_names.add(os.path.basename(entry))
+                    else:
+                        name = os.path.basename(os.path.dirname(entry))
+                        wallet_names.add(name)
+            for wallet_name in wallet_names:
+                self.extract_database_wallet(wallet_name)
+
     def extract_opss_wallet(self):
         """
         Extract the and unzip the OPSS wallet, if present, and return the path to the wallet directory.

core/src/main/resources/oracle/weblogic/deploy/messages/wlsdeploy_rb.properties

@@ -740,6 +740,15 @@ WLSDPLY-06363=Skipping {0} File Store {1}
 WLSDPLY-06364=Discovering {0} {1} elements
 WLSDPLY-06365=Adding {0} {1}
 WLSDPLY-06366={0} configuration for {1} {2} will not be discovered in online mode
+WLSDPLY-06367=Add JDBC Datasource DriverParams wallet based on property {0} into archive entry path {1}
+WLSDPLY-06368=Unable to collect wallet directory because the JDBC Datasource DriverParams connection property value {0} \
+  is at ORACLE_HOME directory {1}. 
+WLSDPLY-06369=Parent directory of JDBC Datasource DriverParams property value {0} is not a flat structure, only \
+  individual file will be added to the wallet, other files that may exists in the original database wallet will not be \
+  collected.
+WLSDPLY-06370=Datasource {0} JDBC Datasource DriverParams property {1} value {2} does not exist in the file system.
+WLSDPLY-06371=Remote discovery will not collect any JDBC Datasource DriverParams connection property that references \
+  file or directory.
 
 # deployments_discoverer.py
 WLSDPLY-06380=Discovering domain model deployments

integration-tests/system-test/src/test/java/oracle/weblogic/deploy/integration/BaseTest.java

@@ -1,4 +1,4 @@
-// Copyright 2019, 2021, Oracle Corporation and/or its affiliates.
+// Copyright 2019, 2023, Oracle Corporation and/or its affiliates.
 // Licensed under the Universal Permissive License v 1.0 as shown at http://oss.oracle.com/licenses/upl.
 
 package oracle.weblogic.deploy.integration;
@@ -33,6 +33,7 @@ public class BaseTest {
     protected static final String FS = File.separator;
     private static final String SAMPLE_ARCHIVE_FILE = "archive.zip";
     private static final String UPDATED_SAMPLE_ARCHIVE_FILE = "archive2.zip";
+    private static final String WALLET_SAMPLE_ARCHIVE_FILE = "archive-jdbc.zip";
     private static final String WDT_ZIPFILE = "weblogic-deploy.zip";
     private static final String WDT_HOME_DIR = "weblogic-deploy";
     protected static final String SAMPLE_MODEL_FILE_PREFIX = "simple-topology";
@@ -49,6 +50,7 @@ public class BaseTest {
     protected static String validateModelScript = "";
     protected static String domainParentDir = "";
     protected static String prepareModelScript = "";
+    protected static String archiveHelperScript = "";
     protected static final String ORACLE_DB_IMG = "phx.ocir.io/weblogick8s/database/enterprise";
     protected static final String ORACLE_DB_IMG_TAG = "12.2.0.1-slim";
     private static final String DB_CONTAINER_NAME = generateDatabaseContainerName();
@@ -90,6 +92,7 @@ protected static void initialize() {
         validateModelScript = getWDTScriptsHome() + FS + "validateModel.sh";
         compareModelScript = getWDTScriptsHome() + FS + "compareModel.sh";
         prepareModelScript = getWDTScriptsHome() + FS + "prepareModel.sh";
+        archiveHelperScript = getWDTScriptsHome() + FS + "archiveHelper.sh";
 
         domainParentDir = "." + FS + "target" + FS + "domains";
     }
@@ -208,6 +211,10 @@ protected static String getSampleArchiveFile() {
         return getTargetDir() + FS + SAMPLE_ARCHIVE_FILE;
     }
 
+    protected static String getSampleArchiveFileWithWallet() {
+        return getTargetDir() + FS + WALLET_SAMPLE_ARCHIVE_FILE;
+    }
+
     protected static String getUpdatedSampleArchiveFile() {
         return getTargetDir() + FS + UPDATED_SAMPLE_ARCHIVE_FILE;
     }
@@ -216,6 +223,10 @@ protected static String getSampleModelFile(String suffix) {
         return getResourcePath() + FS + SAMPLE_MODEL_FILE_PREFIX + suffix + ".yaml";
     }
 
+    protected static String getSampleKeyStoreFile() {
+        return getResourcePath() + FS + "cwallet.sso";
+    }
+
     protected static Path getInstallerTargetDir() {
         return Paths.get("..", "installer", "target");
     }