Various RCU related update (#1193)

* wdt-636  Remove the use of getDatabaseDefaults during JRF domain creation, unify the logic for atp, ssl and regular oracle db for rcu datasources connection parameters setup.

* minor refactoring

* no more need to update password from getDatabaseDefaults - operator mii use case since we are populated the password from the model or cli

* temporary

* testing for MDS

* ATP refactor 1

* ATP refactoring 2

* add AGL

* refactoring

* Move RCUDbInfo to under resources section and rename as RCUConfiguration

* refactor to use RCUConfiguraion

* Fix wlst path issue

* refactoring

* correct NPE in rcudbinfo_helper

* refactor

* bug

* cleanup

* remove RCUConfiguration

* fix keystores password encrypted property for datasource

* fix atp wallet extraction

* Remove new sections, user can use sparse model in regular resources/JDBCSystemResource section for more flexibility and robustness

* doc update

* doc update

* doc change

* cleanup

* doc update

* doc update

* Fix aliases failure, remove obsolete code

* remove obsolete comment

* correct obsolete import

* clean up

Author: jshum2479

core/src/main/java/oracle/weblogic/deploy/create/RCURunner.java

@@ -19,6 +19,7 @@
 import oracle.weblogic.deploy.util.ScriptRunnerException;
 import oracle.weblogic.deploy.util.StringUtils;
 
+import org.python.core.PyClass;
 import org.python.core.PyDictionary;
 import org.python.core.PyString;
 
@@ -135,47 +136,54 @@ public static RCURunner createRunner(String domainType, String oracleHome, Strin
      * @param javaHome   the JAVA_HOME location
      * @param rcuSchemas the list of RCU schemas to create (this list should not include STB)
      * @param rcuVariables a comma separated list of key=value variables
-     * @param rcuProperties dictionary of ATP specific arguments
+     * @param connectionProperties dictionary of ATP specific arguments
      * @throws CreateException if a parameter validation error occurs
      */
-    public static RCURunner createAtpRunner(String domainType, String oracleHome, String javaHome,
-                                            String rcuPrefix, List<String> rcuSchemas, String rcuVariables,
-                                            PyDictionary rcuProperties) throws CreateException {
-
-        String rcuDb = "jdbc:oracle:thin:@" + get(rcuProperties, "tns.alias");
+    public static RCURunner createAtpRunner(String domainType, String oracleHome, String javaHome, String rcuDb,
+                                            List<String> rcuSchemas, String rcuPrefix, String rcuVariables,
+                                            String databaseType, PyDictionary runnerMap,
+                                            PyDictionary connectionProperties) throws CreateException {
 
         RCURunner runner = new RCURunner(domainType, oracleHome, javaHome, rcuDb, rcuPrefix, rcuSchemas, rcuVariables);
 
-        String tnsAdmin = get(rcuProperties, "oracle.net.tns_admin");
-        String keyStorePassword = get(rcuProperties, "javax.net.ssl.keyStorePassword");
-        String trustStorePassword = get(rcuProperties, "javax.net.ssl.trustStorePassword");
-
         StringBuilder sslArgs = new StringBuilder();
-        sslArgs.append("oracle.net.tns_admin=");
-        sslArgs.append(tnsAdmin);
-        sslArgs.append(",oracle.net.ssl_version=1.2");
-        sslArgs.append(",javax.net.ssl.trustStore=");
-        sslArgs.append(tnsAdmin);
-        sslArgs.append("/truststore.jks");
-        sslArgs.append(",javax.net.ssl.trustStoreType=JKS");
-        sslArgs.append(",javax.net.ssl.trustStorePassword=");
-        sslArgs.append(trustStorePassword);
-        sslArgs.append(",javax.net.ssl.keyStore=");
-        sslArgs.append(tnsAdmin);
-        sslArgs.append("/keystore.jks");
-        sslArgs.append(",javax.net.ssl.keyStoreType=JKS");
-        sslArgs.append(",javax.net.ssl.keyStorePassword=");
-        sslArgs.append(keyStorePassword);
-        sslArgs.append(",oracle.jdbc.fanEnabled=false");
-        sslArgs.append(",oracle.net.ssl_server_dn_match=false");
-
-        runner.atpDB = true;
+
+        for (Object connectionProperty: connectionProperties.keys()) {
+            if (sslArgs.length() != 0) {
+                sslArgs.append(',');
+            }
+            sslArgs.append(connectionProperty.toString());
+            sslArgs.append('=');
+            PyDictionary valueObject = (PyDictionary)connectionProperties
+                .get(new PyString(connectionProperty.toString()));
+            sslArgs.append(valueObject.get(new PyString("Value")));
+        }
+
+
+        addExtraSSLPropertyFromMap(runnerMap, connectionProperties, sslArgs, "javax.net.ssl.keyStorePassword");
+        addExtraSSLPropertyFromMap(runnerMap, connectionProperties, sslArgs, "javax.net.ssl.trustStorePassword");
+
+
+        runner.atpDB = true; // "ATP".equals(databaseType);  // or scan if there are any 'ssl' in properties ?
         runner.atpSSlArgs = sslArgs.toString();
-        runner.atpAdminUser = get(rcuProperties, "atp.admin.user");
-        runner.atpDefaultTablespace = get(rcuProperties, "atp.default.tablespace");
-        runner.atpTemporaryTablespace = get(rcuProperties, "atp.temp.tablespace");
+
+        runner.atpAdminUser = get(runnerMap, "atp.admin.user");
+        runner.atpDefaultTablespace = get(runnerMap, "atp.default.tablespace");
+        runner.atpTemporaryTablespace = get(runnerMap, "atp.temp.tablespace");
+
         return runner;
     }
+
+    private static void addExtraSSLPropertyFromMap(PyDictionary runnerMap, PyDictionary connectionProperties,
+                                                   StringBuilder sslArgs, String key) {
+        if (!connectionProperties.has_key(new PyString(key)) &&
+            !get(runnerMap, key).equals("None")) {
+            sslArgs.append(",");
+            sslArgs.append(key);
+            sslArgs.append(get(runnerMap, key));
+        }
+    }
+
     /**
      * Build an RCU runner for an SSL database.
      *
@@ -276,6 +284,7 @@ public void runRcu(String rcuSysPass, String rcuSchemaPass) throws CreateExcepti
         }
         // RCU is stupid and RCU drop exits with exit code 1 if the schemas do not exist...sigh
         //
+
         if (exitCode != 0 && !isSchemaNotExistError(runner)) {
             CreateException ce = new CreateException("WLSDPLY-12002", CLASS, exitCode, runner.getStdoutFileName());
             LOGGER.throwing(CLASS, METHOD, ce);

core/src/main/python/create.py

@@ -48,6 +48,8 @@
 from wlsdeploy.util.weblogic_helper import WebLogicHelper
 from wlsdeploy.tool.create import atp_helper
 from wlsdeploy.tool.create import ssl_helper
+from wlsdeploy.aliases.model_constants import DOMAIN_INFO
+from wlsdeploy.aliases.model_constants import DRIVER_PARAMS_NET_TNS_ADMIN
 
 wlst_helper.wlst_functions = globals()
 
@@ -237,43 +239,47 @@ def validate_rcu_args_and_model(model_context, model, archive_helper, aliases):
 
     has_atpdbinfo = 0
     has_ssldbinfo = 0
-    domain_info = model[model_constants.DOMAIN_INFO]
-    if domain_info is not None:
-        if model_constants.RCU_DB_INFO in domain_info:
-            rcu_db_info = RcuDbInfo(model_context, aliases, domain_info[model_constants.RCU_DB_INFO])
-            has_tns_admin = rcu_db_info.has_tns_admin()
-            has_regular_db = rcu_db_info.is_regular_db()
-            has_atpdbinfo = rcu_db_info.has_atpdbinfo()
-            has_ssldbinfo = rcu_db_info.has_ssldbinfo()
-
-            if archive_helper and not has_regular_db:
-                System.setProperty('oracle.jdbc.fanEnabled', 'false')
-
-                # 1. If it does not have the oracle.net.tns_admin specified, then extract to domain/atpwallet
-                # 2. If it is plain old regular oracle db, do nothing
-                # 3. If it deos not have tns_admin in the model, then the wallet must be in the archive
-                if not has_tns_admin:
-                    wallet_path = archive_helper.extract_atp_wallet()
-                    if wallet_path:
-                        # update the model to add the tns_admin
-                        model[model_constants.DOMAIN_INFO][model_constants.RCU_DB_INFO][
-                            model_constants.DRIVER_PARAMS_NET_TNS_ADMIN] = wallet_path
-                    else:
-                        __logger.severe('WLSDPLY-12411', error=None, class_name=_class_name, method_name=_method_name)
-                        cla_helper.clean_up_temp_files()
-                        tool_exit.end(model_context, ExitCode.ERROR)
-
-        else:
-            if model_context.get_domain_typedef().required_rcu():
-                if not model_context.get_rcu_database() or not model_context.get_rcu_prefix():
-                    __logger.severe('WLSDPLY-12408', model_context.get_domain_type(), CommandLineArgUtil.RCU_DB_SWITCH,
-                                    CommandLineArgUtil.RCU_PREFIX_SWITCH)
-                    cla_helper.clean_up_temp_files()
-                    tool_exit.end(model_context, ExitCode.ERROR)
+
+    if model_constants.DOMAIN_INFO in model and model_constants.RCU_DB_INFO in model[model_constants.DOMAIN_INFO]:
+        rcu_db_info = RcuDbInfo(model_context, aliases, model[model_constants.DOMAIN_INFO][model_constants.RCU_DB_INFO])
+        has_tns_admin = rcu_db_info.has_tns_admin()
+        is_regular_db = rcu_db_info.is_regular_db()
+        has_atpdbinfo = rcu_db_info.has_atpdbinfo()
+        has_ssldbinfo = rcu_db_info.has_ssldbinfo()
+
+        _validate_atp_wallet_in_archive(archive_helper, is_regular_db, has_tns_admin, model,
+                                        model_context)
+    else:
+        if model_context.get_domain_typedef().required_rcu():
+            if not model_context.get_rcu_database() or not model_context.get_rcu_prefix():
+                __logger.severe('WLSDPLY-12408', model_context.get_domain_type(), CommandLineArgUtil.RCU_DB_SWITCH,
+                                CommandLineArgUtil.RCU_PREFIX_SWITCH)
+                cla_helper.clean_up_temp_files()
+                tool_exit.end(model_context, CommandLineArgUtil.PROG_ERROR_EXIT_CODE)
 
     return has_atpdbinfo, has_ssldbinfo
 
 
+def _validate_atp_wallet_in_archive(archive_helper, is_regular_db, has_tns_admin, model, model_context):
+    if archive_helper and not is_regular_db:
+        # 1. If it does not have the oracle.net.tns_admin specified, then extract to domain/atpwallet
+        # 2. If it is plain old regular oracle db, do nothing
+        # 3. If it deos not have tns_admin in the model, then the wallet must be in the archive
+        if not has_tns_admin:
+            wallet_path = archive_helper.extract_atp_wallet()
+            if wallet_path:
+                # update the model to add the tns_admin
+                model[model_constants.DOMAIN_INFO][model_constants.RCU_DB_INFO][
+                    model_constants.DRIVER_PARAMS_NET_TNS_ADMIN] = wallet_path
+            else:
+                __logger.severe('WLSDPLY-12411', error=None, class_name=_class_name, method_name=_method_name)
+                cla_helper.clean_up_temp_files()
+                tool_exit.end(model_context, CommandLineArgUtil.PROG_ERROR_EXIT_CODE)
+
+    if not is_regular_db:
+        System.setProperty('oracle.jdbc.fanEnabled', 'false')
+
+
 def _get_domain_path(model_context, model):
     """
     Returns the domain home path.
@@ -332,16 +338,17 @@ def main(args):
             archive_helper = ArchiveHelper(archive_file_name, domain_path, __logger, ExceptionType.CREATE)
 
         has_atp, has_ssl = validate_rcu_args_and_model(model_context, model_dictionary, archive_helper, aliases)
-
         # check if there is an atpwallet and extract in the domain dir
         # it is to support non JRF domain but user wants to use ATP database
-        if not has_atp and archive_helper:
+        if has_atp and archive_helper:
             archive_helper.extract_atp_wallet()
 
         creator = DomainCreator(model_dictionary, model_context, aliases)
         creator.create()
 
         if has_atp:
+            # if extracted_wallet_path is not None:
+            #     model_dictionary[DOMAIN_INFO][DRIVER_PARAMS_NET_TNS_ADMIN] = extracted_wallet_path
             rcu_properties_map = model_dictionary[model_constants.DOMAIN_INFO][model_constants.RCU_DB_INFO]
             rcu_db_info = RcuDbInfo(model_context, aliases, rcu_properties_map)
             atp_helper.fix_jps_config(rcu_db_info, model_context)

core/src/main/python/wlsdeploy/aliases/model_constants.py

@@ -28,13 +28,16 @@
 RCU_PREFIX = 'rcu_prefix'
 RCU_SCHEMA_PASSWORD = 'rcu_schema_password'
 RCU_ADMIN_PASSWORD = 'rcu_admin_password'
+RCU_DEFAULT_TBLSPACE = 'rcu_default_tablespace'
+RCU_TEMP_TBLSPACE = 'rcu_temp_tablespace'
 RCU_DB_USER = 'rcu_db_user'
 RCU_DB_CONN = 'rcu_db_conn_string'
 RCU_COMP_INFO = 'compInfoXMLLocation'
 RCU_STG_INFO = 'storageXMLLocation'
 RCU_VARIABLES = 'rcu_variables'
+DATABASE_TYPE = 'databaseType'
 USE_ATP = 'useATP'
-ATP_TNS_ENTRY = 'tns.alias'
+TNS_ENTRY = 'tns.alias'
 ATP_DEFAULT_TABLESPACE = 'atp.default.tablespace'
 ATP_TEMPORARY_TABLESPACE = 'atp.temp.tablespace'
 ATP_ADMIN_USER = 'atp.admin.user'
@@ -280,8 +283,6 @@
 SOURCE_DESTINATION = 'SourceDestination'
 SQL_AUTHENTICATOR = 'SQLAuthenticator'
 SSL = 'SSL'
-SSL_ADMIN_USER = 'ssl.admin.user'
-SSL_TNS_ENTRY = 'tns.alias'
 STARTUP_CLASS = 'StartupClass'
 STORE = 'Store'
 SUB_DEPLOYMENT = 'SubDeployment'
@@ -360,7 +361,7 @@
 DRIVER_PARAMS_PROPERTY_VALUE_ENCRYPTED = 'EncryptedValueEncrypted'
 DRIVER_PARAMS_USER_PROPERTY = 'user'
 DRIVER_PARAMS_TRUSTSTORE_PROPERTY = 'javax.net.ssl.trustStore'
-DRIVER_PARAMS_kEYSTORE_PROPERTY = 'javax.net.ssl.keyStore'
+DRIVER_PARAMS_KEYSTORE_PROPERTY = 'javax.net.ssl.keyStore'
 DRIVER_PARAMS_TRUSTSTORETYPE_PROPERTY = 'javax.net.ssl.trustStoreType'
 DRIVER_PARAMS_KEYSTORETYPE_PROPERTY = 'javax.net.ssl.keyStoreType'
 DRIVER_PARAMS_TRUSTSTOREPWD_PROPERTY = 'javax.net.ssl.trustStorePassword'

core/src/main/python/wlsdeploy/tool/create/atp_helper.py

@@ -25,15 +25,15 @@ def set_ssl_properties(xml_doc, atp_creds_path, keystore_password, truststore_pa
 
     for prop in props:
         if prop.getAttribute('name') == 'props.db.1':
+            set_property(dom_tree, prop, 'oracle.net.ssl_server_dn_match', 'true')
+            set_property(dom_tree, prop, 'oracle.net.ssl_version', '1.2')
+            set_property(dom_tree, prop, 'oracle.net.tns_admin', atp_creds_path)
             set_property(dom_tree, prop, 'javax.net.ssl.trustStoreType', 'JKS')
             set_property(dom_tree, prop, 'javax.net.ssl.trustStore', atp_creds_path + '/truststore.jks')
-            set_property(dom_tree, prop, 'oracle.net.tns_admin', atp_creds_path)
             set_property(dom_tree, prop, 'javax.net.ssl.keyStoreType', 'JKS')
             set_property(dom_tree, prop, 'javax.net.ssl.keyStore', atp_creds_path + '/keystore.jks')
             set_property(dom_tree, prop, 'javax.net.ssl.keyStorePassword', keystore_password)
             set_property(dom_tree, prop, 'javax.net.ssl.trustStorePassword', truststore_password)
-            set_property(dom_tree, prop, 'oracle.net.ssl_server_dn_match', 'true')
-            set_property(dom_tree, prop, 'oracle.net.ssl_version', '1.2')
             # Persist the changes in the xml file
             file_handle = open(xml_doc, "w")
             dom_tree.writexml(file_handle)
@@ -56,14 +56,14 @@ def set_property(dom_tree, prop, name, value):
     prop.appendChild(newline)
 
 def fix_jps_config(rcu_db_info, model_context):
-    tns_admin = rcu_db_info.get_atp_tns_admin()
+    tns_admin = rcu_db_info.get_tns_admin()
     keystore_password = rcu_db_info.get_keystore_password()
     truststore_password = rcu_db_info.get_truststore_password()
 
-    jsp_config = model_context.get_domain_home() + '/config/fmwconfig/jps-config.xml'
-    jsp_config_jse = model_context.get_domain_home() + '/config/fmwconfig/jps-config-jse.xml'
-    set_ssl_properties(jsp_config, tns_admin, keystore_password, truststore_password)
-    set_ssl_properties(jsp_config_jse, tns_admin, keystore_password, truststore_password)
+    jps_config = model_context.get_domain_home() + '/config/fmwconfig/jps-config.xml'
+    jps_config_jse = model_context.get_domain_home() + '/config/fmwconfig/jps-config-jse.xml'
+    set_ssl_properties(jps_config, tns_admin, keystore_password, truststore_password)
+    set_ssl_properties(jps_config_jse, tns_admin, keystore_password, truststore_password)
 
 
 def get_atp_connect_string(tnsnames_ora_path, tns_sid_name):