Skip to content
This repository was archived by the owner on Apr 20, 2023. It is now read-only.

Commit 88057f4

Browse files
arunskurianarunskurian
committed
major-iteration-1: schema json changes to remove unused options, better exception handling, addition of database list options, using terminate function instead of die in library, logsync is not hardcoded anymore
1 parent bc6a595 commit 88057f4

File tree

13 files changed

+580
-365
lines changed

13 files changed

+580
-365
lines changed

‎plugin_config.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,4 +7,4 @@ pluginType: STAGED
77
entryPoint: plugin_runner:plugin
88
srcDir: src
99
schemaFile: schema.json
10-
buildNumber: 1.69.9
10+
buildNumber: 1.69.25

‎schema.json‎

Lines changed: 17 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@
5757
"type": "object",
5858
"additionalProperties" : false,
5959
"ordering": [
60-
"vdbHost",
6160
"vdbUser",
6261
"vdbPass",
6362
"baseDir",
@@ -67,7 +66,6 @@
6766
"config_settings_prov"
6867
],
6968
"required": [
70-
"vdbHost",
7169
"vdbUser",
7270
"vdbPass",
7371
"baseDir",
@@ -76,12 +74,6 @@
7674
"mPath"
7775
],
7876
"properties" : {
79-
"vdbHost": {
80-
"type": "string",
81-
"prettyName": "HostName or IP",
82-
"description": "HostName or IP Address of Target Host for VDBs",
83-
"default": "localhost"
84-
},
8577
"vdbUser": {
8678
"type": "string",
8779
"prettyName": "DB User",
@@ -154,36 +146,30 @@
154146
"dSourceType",
155147
"serverId",
156148
"stagingPort",
157-
"mountPath",
158-
"stagingip"
149+
"mountPath"
159150
],
160151
"ordering" : [
161152
"dSourceType",
162-
"stagingip",
163-
"stagingUser",
164-
"stagingPass",
165-
"stagingBasedir",
166153
"serverId",
167154
"stagingPort",
155+
"stagingPass",
156+
"stagingBasedir",
168157
"mountPath",
169-
"sourceip",
170-
"backupPath",
158+
"sourceip",
171159
"sourceUser",
172160
"sourcePass",
161+
"databaseList",
162+
"backupPath",
173163
"logSync",
174164
"replicationUser",
175-
"replicationPass",
176-
"sourceDatabase",
177-
"sourceTables",
178-
"scpUser",
179-
"scpPass"
165+
"replicationPass"
180166
],
181167
"properties" : {
182168
"dSourceType": {
183169
"type": "string",
184170
"prettyName": "dSource Type",
185171
"description": "",
186-
"enum": ["Manual Backup Ingestion", "Replication","Simple (Tablespace Backup)"],
172+
"enum": ["Manual Backup Ingestion", "Replication"],
187173
"default": "Replication"
188174
},
189175
"backupPath": {
@@ -192,12 +178,6 @@
192178
"description": "* Leave blank if Delphix is taking backups",
193179
"default": ""
194180
},
195-
"stagingip": {
196-
"type": "string",
197-
"prettyName": "MySQL Staging Server IP address",
198-
"description": "HostName or IP address of the staging server",
199-
"default": "localhost"
200-
},
201181
"sourceip": {
202182
"type": "string",
203183
"prettyName": "Source DB Host IP address",
@@ -223,9 +203,15 @@
223203
"description": "* Required if Delphix is taking backups",
224204
"default": "Delphix@123"
225205
},
206+
"databaseList": {
207+
"type": "string",
208+
"prettyName": "Databases List",
209+
"description": "If Delphix is taking the backup, list of databases to include (comma separated)",
210+
"default": "ALL"
211+
},
226212
"logSync": {
227213
"default": true,
228-
"prettyName": "LogSync",
214+
"prettyName": "Log Sync",
229215
"type": "boolean",
230216
"description": "LogSync needs to be enabled if using Replication Method."
231217
},
@@ -242,17 +228,11 @@
242228
"description": "* Required if using Replication Method with LogSync",
243229
"default": "Delphix@123"
244230
},
245-
"stagingUser": {
246-
"type": "string",
247-
"prettyName": "Staging DB User Name",
248-
"description": "MySQL Staging DB User Name",
249-
"default": "root"
250-
},
251231
"stagingPass": {
252232
"type": "string",
253233
"format": "password",
254-
"prettyName": "Staging DB Password",
255-
"description": "Staging DB User's Password",
234+
"prettyName": "Staging Initialization Password",
235+
"description": "root user password to use while initializing Staging DB.",
256236
"default": "Delphix@123"
257237
},
258238
"serverId": {
@@ -261,30 +241,6 @@
261241
"description": "ServerID for the Stagine DB. Must be different from Source DB Server ID",
262242
"default": "102"
263243
},
264-
"sourceDatabase": {
265-
"type": "string",
266-
"prettyName": "Source DB",
267-
"description": "* Name of the Source DB to backup. Required for Simple Tablespace Backup Method",
268-
"default": "delphixdb"
269-
},
270-
"sourceTables": {
271-
"type": "string",
272-
"prettyName": "Source Tables to backup",
273-
"description": "* ALL or table names separated by comma. Required for Simple Tablespace Backup Method",
274-
"default": "ALL"
275-
},
276-
"scpUser": {
277-
"type": "string",
278-
"prettyName": "Source scp Username",
279-
"description": "* Required for Simple Tablespace Backup Method",
280-
"default": "delphix"
281-
},
282-
"scpPass": {
283-
"type": "string",
284-
"prettyName": "Source scp Password",
285-
"description": "* Required for Simple Tablespace Backup Method",
286-
"default": "delphix"
287-
},
288244
"stagingPort": {
289245
"type": "string",
290246
"prettyName": "Staging DB Port",

‎src/common/constants.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,51 @@
88

99
# Constants
1010
VAL = ""
11+
12+
13+
ERR_START_MSG= "Unable to start the MySQL database."
14+
ERR_START_ACTION="Please make sure that" \
15+
"1. Delphix user has the necessary permissions on the host " \
16+
"2. Mount location provided is not in use " \
17+
"3. Database credentials provided are accurate " \
18+
"4. You have provided a serverid is greater than 1 "\
19+
"5. Provided port is not in use."
20+
21+
ERR_INVALID_BINARY_MSG="An invalid path to mysql binary was passed wile provisioning."
22+
ERR_INVALID_BINARY_ACTION="mysql was not found under the provided installation directory \n" \
23+
"Please verify the MySQL directory and retry the operation."
24+
25+
ERR_MYCNF_MSG="MySQL configuration file (my.cnf) not found on host."
26+
ERR_MYCNF_ACTION="Please provide a my.cnf file in the Delphix Toolkit folder on host."
27+
28+
ERR_PWD_MSG="An error occurred while changing password for the MySQL Staging database."
29+
ERR_PWD_ACTION="Please check logs for further information on the error."
30+
31+
ERR_RESTORE_MSG="An error occurred while restoring the Staging database from Source backup."
32+
ERR_RESTORE_ACTION="Please check logs for further information on the error."
33+
34+
ERR_CONNECT_MSG="Unable to connect to MySQL database."
35+
ERR_CONNECT_ACTION="Please verify and confirm that " \
36+
"1. The DB username and password are accurate " \
37+
"2. The MySQL Database is running on the Host. " \
38+
"Check logs for further information on the error."
39+
40+
ERR_BACKUP_MSG="Error while creating source database backup."
41+
ERR_BACKUP_ACTION="Please verify and confirm that " \
42+
"1. Staging Host is able to connect to Source Host " \
43+
"2. Source DB credentials are correct and the db user has the required permissions " \
44+
"3. Databases provided in the list are present in source MySQL instance. " \
45+
"if the issue still persists, please contact Delphix."
46+
47+
ERR_CUST_BACKUP_MSG="There was an issue with provided backup file."
48+
ERR_CUST_BACKUP_ACTION="Please verify the backup location and ensure that the file exists and is not empty."
49+
50+
51+
ERR_INVALID_BINARY_MSG="An invalid path to mysql binary was passed wile provisioning."
52+
ERR_INVALID_BINARY_ACTION="mysql was not found under the provided installation directory \n" \
53+
"Please verify the MySQL directory and retry the operation."
54+
55+
ERR_GENERAL_MSG="An error occurred while provisioning the MySQL VDB."
56+
ERR_GENERAL_ACTION="This could be due to an unsupported version of MySQL."
57+
ERR_GENERAL_OUT="For additional details, check the log " \
58+
"files under the toolkit directory on Target Host."

‎src/common/utils.py‎

Lines changed: 120 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
import time
44
from datetime import datetime
55
from dlpx.virtualization import libs
6-
6+
import constants as const
7+
from dlpx.virtualization.platform.exceptions import UserError
78

89
def _setup_logger():
910
# This will log the time, level, filename, line number, and log message.
@@ -28,3 +29,121 @@ def get_current_time():
2829
curr_time = datetime.now()
2930
return curr_time.strftime('%Y%m%d%H%M%S')
3031

32+
def process_exit_codes(exit_code,operation,std_err=None):
33+
"""
34+
Processes exit code and returns a UserError
35+
Args:
36+
exit_code: Exit code from run_bash
37+
operation: The operation that was performed.
38+
Returns:
39+
UserError
40+
"""
41+
err_out=const.ERR_GENERAL_OUT
42+
if std_err:
43+
err_out= remove_nonascii(std_err)
44+
if exit_code == 3: # Unable to start MySQL
45+
err_msg=const.ERR_START_MSG
46+
err_action=const.ERR_START_ACTION
47+
elif exit_code == 4: # Could not find my.cnf
48+
err_msg=const.ERR_MYCNF_MSG
49+
err_action=const.ERR_MYCNF_ACTION
50+
elif exit_code == 5: # Unable to change password
51+
err_msg=const.ERR_PWD_MSG
52+
err_action=const.ERR_PWD_ACTION
53+
elif exit_code == 6: # Unable restore backup
54+
err_msg=const.ERR_RESTORE_MSG
55+
err_action=const.ERR_RESTORE_ACTION
56+
elif exit_code == 7: # Unable to connect after backup
57+
err_msg=const.ERR_CONNECT_MSG
58+
err_action=const.ERR_CONNECT_ACTION
59+
elif exit_code == 8: # Delphix Backup failed.
60+
err_msg=const.ERR_BACKUP_MSG
61+
err_action=const.ERR_BACKUP_ACTION
62+
elif exit_code == 9: # Delphix Backup failed.
63+
err_msg=const.ERR_CUST_BACKUP_MSG
64+
err_action=const.ERR_CUST_BACKUP_ACTION
65+
elif exit_code == 10: # Invalid Binary Path
66+
err_msg=const.ERR_INVALID_BINARY_MSG
67+
err_action=const.ERR_INVALID_BINARY_ACTION
68+
elif exit_code == 2:
69+
err_msg=const.ERR_GENERAL_MSG
70+
err_action=const.ERR_GENERAL_ACTION
71+
72+
user_error= UserError(
73+
err_msg,
74+
err_action,
75+
"ExitCode:{} \n {}".format(exit_code,err_out)
76+
)
77+
return user_error
78+
79+
80+
def validate_repository(repo):
81+
"""
82+
Sanity check to validate the repositoy string.
83+
Args:
84+
repo (string): Repository path
85+
Returns:
86+
isvalid (bool): Indicates if a repository is valid or not.
87+
"""
88+
if (repo is None or repo.strip()=="" or not repo.strip().startswith("/")):
89+
return False
90+
else:
91+
return True
92+
93+
def parse_db_list(dbstr):
94+
"""
95+
Parse the customer provided db list and convert to space separated string
96+
Args:
97+
dbstr (string): comma separated dblist
98+
Returns:
99+
dbs (string) : space separated dblist
100+
"""
101+
dbs=""
102+
if dbstr is None or dbstr.strip()=="" or dbstr.strip().upper()=="ALL":
103+
dbs= "ALL"
104+
else:
105+
dbstr = dbstr.strip()
106+
dblist = dbstr.split(",")
107+
for db in dblist:
108+
db=db.strip()
109+
dbs+=db
110+
dbs+=" "
111+
dbs+= "mysql"
112+
dbs = dbs.strip()
113+
return dbs
114+
115+
def create_backup_options(logsync, dbs, logger):
116+
"""
117+
Creates the backup options string to be used in restore.sh script for source backup
118+
Args:
119+
logsync (string) : Are we enabling replication?
120+
dbs (string): Which databases are we backing up?
121+
Returns:
122+
backup_options (string): Complete backup options string
123+
"""
124+
backup_options="--skip-lock-tables --single-transaction --flush-logs --hex-blob --no-tablespaces"
125+
try:
126+
if logsync=="true":
127+
backup_options+=" --master-data=2"
128+
if dbs is None or dbs=="ALL":
129+
backup_options+=" -A"
130+
else:
131+
backup_options+=" --databases {}".format(dbs)
132+
logger.debug("Returning backup_options:"+backup_options)
133+
return backup_options
134+
except Exception as err:
135+
user_error= UserError(
136+
"An exception occurred in create_backup_options() while creating DBOptions string for backup",
137+
"Please verify the databases list provided",
138+
"ExitCode:{} \n {}".format("11","Input values are / logsync:{} and dbs:{}".format(logsync,dbs))
139+
)
140+
141+
def remove_nonascii(string_nonASCII):
142+
"""
143+
Removes the non ascii chars from passed string
144+
"""
145+
if not string_nonASCII:
146+
return ""
147+
string_encode = string_nonASCII.encode("ascii", "ignore")
148+
string_decode = string_encode.decode()
149+
return string_decode

‎src/plugin_runner.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -132,16 +132,16 @@ def virtual_post_snapshot(virtual_source, repository, source_config):
132132
logger.debug("Started VDB")
133133
snapshot = SnapshotDefinition(validate=False)
134134
snapshot.snapshot_id= str(utils.get_snapshot_id())
135-
snapshot.snap_host=virtual_source.parameters.vdb_host
135+
snapshot.snap_host=virtual_source.connection.environment.host.name
136136
snapshot.snap_port=virtual_source.parameters.port
137-
snapshot.snap_data_dir=virtual_source.mounts[0].mount_path
137+
snapshot.snap_data_dir=virtual_source.mounts[0].mount_path+"/data"
138138
snapshot.snap_base_dir=virtual_source.parameters.base_dir
139139
snapshot.snap_pass=virtual_source.parameters.vdb_pass
140-
snapshot.snap_backup_path="" # Review this with Ajay
140+
snapshot.snap_backup_path=""
141141
snapshot.snap_time=utils.get_current_time()
142142
logger.debug("SnapShot Definition Created")
143143
logger.debug(snapshot)
144-
return SnapshotDefinition()
144+
return snapshot
145145

146146
@plugin.virtual.start()
147147
def start(virtual_source, repository, source_config):
@@ -153,7 +153,14 @@ def start(virtual_source, repository, source_config):
153153
@plugin.virtual.stop()
154154
def stop(virtual_source, repository, source_config):
155155
logger.debug("virtual.stop > Start")
156-
pluginops.stop_mysql(virtual_source.parameters.port,virtual_source.connection,virtual_source.parameters.base_dir,virtual_source.parameters.vdb_user,virtual_source.parameters.vdb_pass,virtual_source.parameters.vdb_host)
156+
pluginops.stop_mysql(
157+
virtual_source.parameters.port,
158+
virtual_source.connection,
159+
virtual_source.parameters.base_dir,
160+
virtual_source.parameters.vdb_user,
161+
virtual_source.parameters.vdb_pass,
162+
"localhost"
163+
)
157164
logger.debug("virtual.stop > End")
158165
virtual_status(virtual_source, repository, source_config)
159166

0 commit comments

Comments
 (0)