Skip to content

Commit 0525160

Browse files
committed
Improve directory creation logic and logging in binlog_replicator.py
- Refactored directory creation handling to ensure robust creation of parent directories, preventing potential startup failures. - Enhanced logging for directory creation errors to provide clearer diagnostics during execution. - Cleaned up whitespace for better code readability.
1 parent a988687 commit 0525160

File tree

1 file changed

+22
-14
lines changed

1 file changed

+22
-14
lines changed

mysql_ch_replicator/binlog_replicator.py

Lines changed: 22 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ def read_next_event(self) -> LogEvent:
101101

102102
def get_existing_file_nums(data_dir, db_name):
103103
db_path = os.path.join(data_dir, db_name)
104-
104+
105105
# CRITICAL FIX: Always try to create the full directory hierarchy first
106106
# This handles the case where intermediate directories don't exist
107107
try:
@@ -114,22 +114,24 @@ def get_existing_file_nums(data_dir, db_name):
114114
except OSError as e:
115115
# If makedirs fails, try creating step by step
116116
logger.warning(f"Failed to create {db_path} in one step: {e}")
117-
117+
118118
# Find the deepest existing parent directory
119119
current_path = db_path
120120
missing_paths = []
121-
122-
while current_path and current_path != '/' and not os.path.exists(current_path):
121+
122+
while current_path and current_path != "/" and not os.path.exists(current_path):
123123
missing_paths.append(current_path)
124124
current_path = os.path.dirname(current_path)
125-
125+
126126
# Create directories from deepest existing to the target
127127
for path_to_create in reversed(missing_paths):
128128
try:
129129
os.makedirs(path_to_create, exist_ok=True)
130130
logger.debug(f"Created directory: {path_to_create}")
131131
except OSError as create_error:
132-
logger.error(f"Failed to create directory {path_to_create}: {create_error}")
132+
logger.error(
133+
f"Failed to create directory {path_to_create}: {create_error}"
134+
)
133135
raise
134136
existing_files = os.listdir(db_path)
135137
existing_files = [f for f in existing_files if f.endswith(".bin")]
@@ -311,17 +313,19 @@ def get_or_create_file_writer(self, db_name: str) -> FileWriter:
311313

312314
def create_file_writer(self, db_name: str) -> FileWriter:
313315
next_free_file = self.get_next_file_name(db_name)
314-
316+
315317
# Ensure parent directory exists before creating file
316318
parent_dir = os.path.dirname(next_free_file)
317319
if parent_dir:
318320
try:
319321
os.makedirs(parent_dir, exist_ok=True)
320322
logger.debug(f"Ensured directory exists for binlog file: {parent_dir}")
321323
except OSError as e:
322-
logger.error(f"Critical: Failed to create binlog file directory {parent_dir}: {e}")
324+
logger.error(
325+
f"Critical: Failed to create binlog file directory {parent_dir}: {e}"
326+
)
323327
raise
324-
328+
325329
return FileWriter(next_free_file)
326330

327331
def get_next_file_name(self, db_name: str):
@@ -377,19 +381,23 @@ def load(self):
377381

378382
def save(self):
379383
file_name = self.file_name
380-
384+
381385
# Ensure parent directory exists before saving - handles nested isolation paths
382386
parent_dir = os.path.dirname(file_name)
383387
if parent_dir: # Only proceed if there's actually a parent directory
384388
try:
385389
# Use makedirs with exist_ok=True to create all directories recursively
386390
# This handles nested isolation paths like /app/binlog/w2_7cf22b01
387391
os.makedirs(parent_dir, exist_ok=True)
388-
logger.debug(f"Ensured directory exists for binlog state file: {parent_dir}")
392+
logger.debug(
393+
f"Ensured directory exists for binlog state file: {parent_dir}"
394+
)
389395
except OSError as e:
390-
logger.error(f"Critical: Failed to create binlog state directory {parent_dir}: {e}")
396+
logger.error(
397+
f"Critical: Failed to create binlog state directory {parent_dir}: {e}"
398+
)
391399
raise
392-
400+
393401
data = json.dumps(
394402
{
395403
"last_seen_transaction": self.last_seen_transaction,
@@ -521,7 +529,7 @@ def run(self):
521529

522530
self.update_state_if_required(transaction_id)
523531

524-
logger.debug(f"received event {type(event)}, {transaction_id}")
532+
# logger.debug(f"received event {type(event)}, {transaction_id}")
525533

526534
if type(event) not in (
527535
DeleteRowsEvent,

0 commit comments

Comments
 (0)