add option --data-path for mysqldump

Luqun Lou · Herman Lee · commit f00dcb607951 · 2023-10-03T09:48:01.000-07:00
Summary:
mysqldump support dump each table as two file(.sql and .txt) into same directory.

This new options is to allow dump tables into two directory: .sql directory and .txt direory.  .sql directory is specified in --tab while .txt directory is specified in --data-path directory.

Reviewed By: lth

Differential Revision: D45404355
diff --git a/client/client_priv.h b/client/client_priv.h
@@ -210,6 +210,7 @@ enum options_client {
   OPT_COMPRESS_DATA_CHUNK_SIZE,
   OPT_ORDER_BY_PRIMARY_FORCE_INDEX,
   OPT_READ_FROM_BINLOG_SERVER,
+  OPT_DATA_PATH,
   /* Add new option above this */
   OPT_MAX_CLIENT_OPTION
 };
diff --git a/client/mysqldump.cc b/client/mysqldump.cc
@@ -157,6 +157,7 @@ static bool ansi_quotes_mode = false;
 
 static uint opt_zstd_compress_level = default_zstd_compression_level;
 static char *opt_compress_algorithm = nullptr;
+static char *opt_data_path = nullptr;
 
 #define MYSQL_OPT_SOURCE_DATA_EFFECTIVE_SQL 1
 #define MYSQL_OPT_SOURCE_DATA_COMMENTED_SQL 2
@@ -783,6 +784,12 @@ static struct my_option my_long_options[] = {
      "Split compressed data on chunks of specified size in megabytes",
      &opt_compression_chunk_size, &opt_compression_chunk_size, 0, GET_ULL,
      OPT_ARG, 0, 0, (ulonglong)(~(my_off_t)0), nullptr, 0, nullptr},
+    {"data-path", OPT_DATA_PATH,
+     "Store data textfile for each table to specificed data-path. if specified,"
+     ".txt file will be stored in specified data-path, while .sql will be "
+     "stored in --tab path.",
+     &opt_data_path, &opt_data_path, nullptr, GET_STR, OPT_ARG, 0, 0, 0,
+     nullptr, 0, nullptr},
     {nullptr, 0, nullptr, nullptr, nullptr, nullptr, GET_NO_ARG, NO_ARG, 0, 0,
      0, nullptr, 0, nullptr}};
 
@@ -1317,6 +1324,11 @@ static int get_options(int *argc, char ***argv) {
     return (EX_USAGE);
   }
 
+  if (opt_data_path && !path) {
+    // NO_LINT_DEBUG
+    fprintf(stderr, "%s: --data-path must be used with --tab.\n", my_progname);
+    return (EX_USAGE);
+  }
   return (0);
 } /* get_options */
 
@@ -4041,21 +4053,25 @@ static void dump_table(char *table, char *db) {
   if (path) {
     char filename[FN_REFLEN], tmp_path[FN_REFLEN];
 
-    /*
-      Convert the path to native os format
-      and resolve to the full filepath.
-    */
-    convert_dirname(tmp_path, path, NullS);
-    my_load_path(tmp_path, tmp_path, nullptr);
-    fn_format(filename, table, tmp_path, ".txt",
-              MYF(MY_UNPACK_FILENAME | MY_APPEND_EXT));
-
-    /* Must delete the file that 'INTO OUTFILE' will write to */
-    my_delete(filename, MYF(0));
+    if (opt_data_path) {
+      /* construct data file full path */
+      fn_format(filename, table, opt_data_path, ".txt", MYF(MY_APPEND_EXT));
+    } else {
+      /*
+          Convert the path to native os format
+          and resolve to the full filepath.
+      */
+      convert_dirname(tmp_path, path, NullS);
+      my_load_path(tmp_path, tmp_path, nullptr);
+      fn_format(filename, table, tmp_path, ".txt",
+                MYF(MY_UNPACK_FILENAME | MY_APPEND_EXT));
 
-    /* convert to a unix path name to stick into the query */
-    to_unix_path(filename);
+      /* Must delete the file that 'INTO OUTFILE' will write to */
+      my_delete(filename, MYF(0));
 
+      /* convert to a unix path name to stick into the query */
+      to_unix_path(filename);
+    }
     /* now build the query string */
 
     dynstr_append_checked(&query_string, "SELECT /*!40001 SQL_NO_CACHE */ ");
diff --git a/mysql-test/r/mysqldump-data-path.result b/mysql-test/r/mysqldump-data-path.result
@@ -0,0 +1,55 @@
+CREATE DATABASE database1;
+USE database1;
+CREATE TABLE t1(a INT PRIMARY KEY, b INT UNIQUE) ENGINE = INNODB;
+INSERT INTO t1 VALUES (1, 3);
+INSERT INTO t1 VALUES (2, 2);
+INSERT INTO t1 VALUES (3, 1);
+==== mysqldump with --data-path ====
+DROP TABLE t1;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t1.txt' INTO TABLE t1;;
+DROP TABLE t1;
+CREATE TABLE t2(x VARCHAR(100));
+CREATE TABLE t3(x VARCHAR(100));
+  20000   20000 1620000
+ 200000  200000 16200000
+LOAD DATA LOCAL INFILE 'VARDIR/t2.tmp' INTO TABLE t2;;
+LOAD DATA LOCAL INFILE 'VARDIR/t3.tmp' INTO TABLE t3;;
+DROP TABLE t2, t3;
+t2.txt.0.zst
+t2.txt.1.zst
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t2.txt.0.zst' INTO TABLE t2 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t2.txt.1.zst' INTO TABLE t2 compressed;
+t3.txt.0.zst
+t3.txt.1.zst
+t3.txt.10.zst
+t3.txt.11.zst
+t3.txt.12.zst
+t3.txt.13.zst
+t3.txt.14.zst
+t3.txt.15.zst
+t3.txt.2.zst
+t3.txt.3.zst
+t3.txt.4.zst
+t3.txt.5.zst
+t3.txt.6.zst
+t3.txt.7.zst
+t3.txt.8.zst
+t3.txt.9.zst
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.0.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.1.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.2.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.3.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.4.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.5.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.6.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.7.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.8.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.9.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.10.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.11.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.12.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.13.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.14.zst' INTO TABLE t3 compressed;
+LOAD DATA LOCAL INFILE 'VARDIR/.txt/t3.txt.15.zst' INTO TABLE t3 compressed;
+DROP TABLE t2, t3;
+DROP DATABASE database1;
diff --git a/mysql-test/t/mysqldump-data-path-master.opt b/mysql-test/t/mysqldump-data-path-master.opt
@@ -0,0 +1,2 @@
+--local-infile=ON
+--secure-file-priv=""
diff --git a/mysql-test/t/mysqldump-data-path.test b/mysql-test/t/mysqldump-data-path.test
@@ -0,0 +1,121 @@
+CREATE DATABASE database1;
+USE database1;
+
+#
+# small table, without compression and chunk
+# 
+CREATE TABLE t1(a INT PRIMARY KEY, b INT UNIQUE) ENGINE = INNODB;
+
+INSERT INTO t1 VALUES (1, 3);
+INSERT INTO t1 VALUES (2, 2);
+INSERT INTO t1 VALUES (3, 1);
+
+--exec mkdir $MYSQL_TMP_DIR/.sql
+--exec mkdir $MYSQL_TMP_DIR/.txt
+
+--echo ==== mysqldump with --data-path ====
+--replace_regex /-- Server version.*/-- SERVER VERSION/ /-- MySQL dump.*[)]/-- MYSQLDUMP VERSION/
+--exec $MYSQL_DUMP --tab=$MYSQL_TMP_DIR/.sql --data-path=$MYSQL_TMP_DIR/.txt  database1
+
+--let $checksum1_1=`CHECKSUM TABLE t1`
+
+DROP TABLE t1;
+
+--exec $MYSQL --disable-comments database1 2>&1 < "$MYSQL_TMP_DIR/.sql/t1.sql"
+
+--replace_result $MYSQL_TMP_DIR VARDIR 
+--eval LOAD DATA LOCAL INFILE '$MYSQL_TMP_DIR/.txt/t1.txt' INTO TABLE t1;
+--let $checksum1_2=`CHECKSUM TABLE t1`
+if($checksum1_1 != $checksum1_2)
+{
+  --echo "table t1 checksums do no not match: [$checksum1] != [$checksum2]"
+}
+# cleanup
+DROP TABLE t1;
+--force-rmdir $MYSQL_TMP_DIR/.sql
+--force-rmdir $MYSQL_TMP_DIR/.txt
+
+#
+# big table with compress and chunk
+#
+CREATE TABLE t2(x VARCHAR(100));
+CREATE TABLE t3(x VARCHAR(100));
+
+# generate random data
+--exec dd bs=20000000 count=1 if=/dev/urandom | base64 -w 80 > $MYSQL_TMP_DIR/t.tmp
+--exec head -n 20000  $MYSQL_TMP_DIR/t.tmp | tee $MYSQL_TMP_DIR/t2.tmp | wc
+--exec head -n 200000  $MYSQL_TMP_DIR/t.tmp | tee $MYSQL_TMP_DIR/t3.tmp | wc
+--remove_file $MYSQL_TMP_DIR/t.tmp
+
+# load data into table
+--replace_result $MYSQL_TMP_DIR VARDIR 
+--eval LOAD DATA LOCAL INFILE '$MYSQL_TMP_DIR/t2.tmp' INTO TABLE t2;
+--replace_result $MYSQL_TMP_DIR VARDIR 
+--eval LOAD DATA LOCAL INFILE '$MYSQL_TMP_DIR/t3.tmp' INTO TABLE t3;
+--remove_file $MYSQL_TMP_DIR/t2.tmp
+--remove_file $MYSQL_TMP_DIR/t3.tmp
+
+# dump input compressed with zstd
+--exec mkdir $MYSQL_TMP_DIR/.sql
+--exec mkdir $MYSQL_TMP_DIR/.txt
+--exec $MYSQL_DUMP --tab=$MYSQL_TMP_DIR/.sql database1 --compress-data --compress-data-chunk-size=1 --data-path=$MYSQL_TMP_DIR/.txt
+
+--let $checksum2_1=`CHECKSUM TABLE t2`
+--let $t2_cnt = `SELECT COUNT(*) from t2` 
+--let $checksum3_1=`CHECKSUM TABLE t3`
+--let $t3_cnt = `SELECT COUNT(*) from t3` 
+
+DROP TABLE t2, t3;
+
+# creat t2
+--exec $MYSQL --disable-comments database1 2>&1 < "$MYSQL_TMP_DIR/.sql/t2.sql"
+
+# load t2 data
+--list_files_write_file $MYSQL_TMP_DIR/t2_filelist $MYSQL_TMP_DIR/.txt/ t2.txt.*.zst
+--cat_file $MYSQL_TMP_DIR/t2_filelist
+--let $count = 2
+--let $i = 0
+while ($i < $count) {
+--replace_result $MYSQL_TMP_DIR VARDIR 
+--eval LOAD DATA LOCAL INFILE '$MYSQL_TMP_DIR/.txt/t2.txt.$i.zst' INTO TABLE t2 compressed
+--inc $i
+}
+# check t2 data
+--let $checksum2_2=`CHECKSUM TABLE t2`
+if($checksum2_1 != $checksum2_2)
+{
+  --echo "table t2 checksums do no not match: [$checksum2_1] != [$checksum2_2]"
+}
+
+# create t3
+--exec $MYSQL --disable-comments database1 2>&1 < "$MYSQL_TMP_DIR/.sql/t3.sql"
+# load data into t3
+--list_files_write_file $MYSQL_TMP_DIR/t3_filelist $MYSQL_TMP_DIR/.txt/ t3.txt.*.zst
+--cat_file $MYSQL_TMP_DIR/t3_filelist
+--let $count = 16
+--let $i = 0
+while ($i < $count) {
+--replace_result $MYSQL_TMP_DIR VARDIR 
+--eval LOAD DATA LOCAL INFILE '$MYSQL_TMP_DIR/.txt/t3.txt.$i.zst' INTO TABLE t3 compressed
+--inc $i
+}
+# check t3 data
+--let $checksum3_2=`CHECKSUM TABLE t3`
+if($checksum3_1 != $checksum3_2)
+{
+  --echo "table t3 checksums do no not match: [$checksum3_1] != [$checksum3_2]"
+#  --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
+#   eval SELECT * FROM t3 INTO OUTFILE '$MYSQL_TMP_DIR/B11764517.tmp';
+}
+--let $t3_cnt_2 = `SELECT COUNT(*) from t3` 
+if($t3_cnt != $t3_cnt_2)
+{
+  --echo "table t3 count do no not match: [$t3_cnt] != [$t3_cnt_2]"
+}
+# cleanup
+DROP TABLE t2, t3;
+--force-rmdir $MYSQL_TMP_DIR/.sql
+--force-rmdir $MYSQL_TMP_DIR/.txt
+--remove_file $MYSQL_TMP_DIR/t2_filelist
+--remove_file $MYSQL_TMP_DIR/t3_filelist
+DROP DATABASE database1;

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+--local-infile=ON`
	`2`	`+--secure-file-priv=""`