Skip to content

Commit ecd9c42

Browse files
nikicnikic
committed
Fix bug #80107: Handling of large compressed packets
There's two layers of packet splitting going on. First, packets need to be split into having a payload of exactly 2^24-1 bytes or being the last packet. If the split packet has size between 2^24-5 and 2^24-1 bytes, the compressed packets also needs to be split, though the choice of split doesn't matter here. I'm splitting off the first 8192 bytes, as that's what I observe libmysqlclient to be doing.
1 parent c7ceebc commit ecd9c42

File tree

3 files changed

+81
-48
lines changed

3 files changed

+81
-48
lines changed

‎NEWS‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,8 @@ PHP NEWS
55
- MySQLnd:
66
. Fixed bug #80115 (mysqlnd.debug doesn't recognize absolute paths with
77
slashes). (cmb)
8+
. Fixed bug #80107 (mysqli_query() fails for ~16 MB long query when
9+
compression is enabled). (Nikita)
810

911
- OPcache:
1012
. Fixed bug #80083 (Optimizer pass 6 removes variables used for ibm_db2 data

‎ext/mysqli/tests/mysqli_real_connect_compression_error.phpt‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,25 @@
11
--TEST--
22
Bug #80107 mysqli_query() fails for ~16 MB long query when compression is enabled
3-
--XFAIL--
4-
The second INSERT query fails with MySQL server has gone away
53
--SKIPIF--
64
<?php
75
require_once('skipif.inc');
86
require_once('skipifemb.inc');
9-
require_once('skipifconnectfailure.inc');
7+
require_once("connect.inc");
8+
$link = @my_mysqli_connect($host, $user, $passwd, $db, $port, $socket);
9+
if (!$link) {
10+
die(sprintf("skip Can't connect to MySQL Server - [%d] %s", mysqli_connect_errno(), mysqli_connect_error()));
11+
}
12+
$result = $link->query("SHOW VARIABLES LIKE 'max_allowed_packet'");
13+
if ($result->fetch_assoc()['Value'] < 0xffffff) {
14+
die('skip max_allowed_packet is less than 0xffffff');
15+
}
1016
?>
11-
--INI--
12-
mysqli.allow_local_infile=1
1317
--FILE--
1418
<?php
1519

16-
include("connect.inc");
20+
require_once("connect.inc");
21+
22+
$data_size = 16777174;
1723

1824
// Insert with compression disabled:
1925

@@ -22,7 +28,7 @@ $result = my_mysqli_real_connect($mysqli, $host, $user, $passwd, $db, $port, $so
2228
$mysqli->query("DROP TABLE IF EXISTS test");
2329
$mysqli->query("CREATE TABLE test (`blob` LONGBLOB NOT NULL)");
2430

25-
$data = str_repeat("x", 16777174);
31+
$data = str_repeat("x", $data_size);
2632
$mysqli->query("INSERT INTO $db.test(`blob`) VALUE ('$data')");
2733

2834
var_dump(mysqli_error_list($mysqli));
@@ -33,7 +39,7 @@ $mysqli->close();
3339
$mysqli = mysqli_init();
3440
$result = my_mysqli_real_connect($mysqli, $host, $user, $passwd, $db, $port, $socket, MYSQLI_CLIENT_COMPRESS);
3541

36-
$data = str_repeat("x", 16777174);
42+
$data = str_repeat("x", $data_size);
3743
$mysqli->query("INSERT INTO $db.test(`blob`) VALUE ('$data')");
3844

3945
var_dump(mysqli_error_list($mysqli));

‎ext/mysqlnd/mysqlnd_protocol_frame_codec.c‎

Lines changed: 65 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,57 @@ MYSQLND_METHOD(mysqlnd_pfc, reset)(MYSQLND_PFC * const pfc, MYSQLND_STATS * cons
5050
#define STORE_HEADER_SIZE(safe_storage, buffer) COPY_HEADER((safe_storage), (buffer))
5151
#define RESTORE_HEADER_SIZE(buffer, safe_storage) STORE_HEADER_SIZE((safe_storage), (buffer))
5252

53+
#ifdef MYSQLND_COMPRESSION_ENABLED
54+
static size_t write_compressed_packet(
55+
const MYSQLND_PFC *pfc, MYSQLND_VIO *vio,
56+
MYSQLND_STATS *conn_stats, MYSQLND_ERROR_INFO *error_info,
57+
zend_uchar *uncompressed_payload, size_t to_be_sent, zend_uchar *compress_buf) {
58+
DBG_ENTER("write_compressed_packet");
59+
/* here we need to compress the data and then write it, first comes the compressed header */
60+
size_t tmp_complen = to_be_sent;
61+
size_t payload_size;
62+
if (PASS == pfc->data->m.encode((compress_buf + COMPRESSED_HEADER_SIZE + MYSQLND_HEADER_SIZE), &tmp_complen,
63+
uncompressed_payload, to_be_sent))
64+
{
65+
int3store(compress_buf + MYSQLND_HEADER_SIZE, to_be_sent);
66+
payload_size = tmp_complen;
67+
} else {
68+
int3store(compress_buf + MYSQLND_HEADER_SIZE, 0);
69+
memcpy(compress_buf + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, uncompressed_payload, to_be_sent);
70+
payload_size = to_be_sent;
71+
}
72+
73+
int3store(compress_buf, payload_size);
74+
int1store(compress_buf + 3, pfc->data->compressed_envelope_packet_no);
75+
DBG_INF_FMT("writing "MYSQLND_SZ_T_SPEC" bytes to the network", payload_size + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE);
76+
77+
size_t bytes_sent = vio->data->m.network_write(vio, compress_buf, payload_size + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, conn_stats, error_info);
78+
pfc->data->compressed_envelope_packet_no++;
79+
#ifdef WHEN_WE_NEED_TO_CHECK_WHETHER_COMPRESSION_WORKS_CORRECTLY
80+
if (res == Z_OK) {
81+
size_t decompressed_size = left + MYSQLND_HEADER_SIZE;
82+
zend_uchar * decompressed_data = mnd_malloc(decompressed_size);
83+
int error = pfc->data->m.decode(decompressed_data, decompressed_size,
84+
compress_buf + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, payload_size);
85+
if (error == Z_OK) {
86+
int i;
87+
DBG_INF("success decompressing");
88+
for (i = 0 ; i < decompressed_size; i++) {
89+
if (i && (i % 30 == 0)) {
90+
printf("\n\t\t");
91+
}
92+
printf("%.2X ", (int)*((char*)&(decompressed_data[i])));
93+
DBG_INF_FMT("%.2X ", (int)*((char*)&(decompressed_data[i])));
94+
}
95+
} else {
96+
DBG_INF("error decompressing");
97+
}
98+
mnd_free(decompressed_data);
99+
}
100+
#endif /* WHEN_WE_NEED_TO_CHECK_WHETHER_COMPRESSION_WORKS_CORRECTLY */
101+
DBG_RETURN(bytes_sent);
102+
}
103+
#endif
53104

54105
/* {{{ mysqlnd_pfc::send */
55106
/*
@@ -91,53 +142,27 @@ MYSQLND_METHOD(mysqlnd_pfc, send)(MYSQLND_PFC * const pfc, MYSQLND_VIO * const v
91142
DBG_INF_FMT("packet_no=%u", pfc->data->packet_no);
92143
#ifdef MYSQLND_COMPRESSION_ENABLED
93144
if (pfc->data->compressed == TRUE) {
94-
/* here we need to compress the data and then write it, first comes the compressed header */
95-
size_t tmp_complen = to_be_sent;
96-
size_t payload_size;
97145
zend_uchar * uncompressed_payload = p; /* should include the header */
98-
99146
STORE_HEADER_SIZE(safe_storage, uncompressed_payload);
100147
int3store(uncompressed_payload, to_be_sent);
101148
int1store(uncompressed_payload + 3, pfc->data->packet_no);
102-
if (PASS == pfc->data->m.encode((compress_buf + COMPRESSED_HEADER_SIZE + MYSQLND_HEADER_SIZE), &tmp_complen,
103-
uncompressed_payload, to_be_sent + MYSQLND_HEADER_SIZE))
104-
{
105-
int3store(compress_buf + MYSQLND_HEADER_SIZE, to_be_sent + MYSQLND_HEADER_SIZE);
106-
payload_size = tmp_complen;
149+
if (to_be_sent <= MYSQLND_MAX_PACKET_SIZE - MYSQLND_HEADER_SIZE) {
150+
bytes_sent = write_compressed_packet(
151+
pfc, vio, conn_stats, error_info,
152+
uncompressed_payload, to_be_sent + MYSQLND_HEADER_SIZE, compress_buf);
107153
} else {
108-
int3store(compress_buf + MYSQLND_HEADER_SIZE, 0);
109-
memcpy(compress_buf + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, uncompressed_payload, to_be_sent + MYSQLND_HEADER_SIZE);
110-
payload_size = to_be_sent + MYSQLND_HEADER_SIZE;
154+
/* The uncompressed size including the header would overflow. Split into two
155+
* compressed packets. The size of the first one is relatively arbitrary here. */
156+
const size_t split_off_bytes = 8192;
157+
bytes_sent = write_compressed_packet(
158+
pfc, vio, conn_stats, error_info,
159+
uncompressed_payload, split_off_bytes, compress_buf);
160+
bytes_sent = write_compressed_packet(
161+
pfc, vio, conn_stats, error_info,
162+
uncompressed_payload + split_off_bytes,
163+
to_be_sent + MYSQLND_HEADER_SIZE - split_off_bytes, compress_buf);
111164
}
112165
RESTORE_HEADER_SIZE(uncompressed_payload, safe_storage);
113-
114-
int3store(compress_buf, payload_size);
115-
int1store(compress_buf + 3, pfc->data->packet_no);
116-
DBG_INF_FMT("writing "MYSQLND_SZ_T_SPEC" bytes to the network", payload_size + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE);
117-
bytes_sent = vio->data->m.network_write(vio, compress_buf, payload_size + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, conn_stats, error_info);
118-
pfc->data->compressed_envelope_packet_no++;
119-
#if WHEN_WE_NEED_TO_CHECK_WHETHER_COMPRESSION_WORKS_CORRECTLY
120-
if (res == Z_OK) {
121-
size_t decompressed_size = left + MYSQLND_HEADER_SIZE;
122-
zend_uchar * decompressed_data = mnd_malloc(decompressed_size);
123-
int error = pfc->data->m.decode(decompressed_data, decompressed_size,
124-
compress_buf + MYSQLND_HEADER_SIZE + COMPRESSED_HEADER_SIZE, payload_size);
125-
if (error == Z_OK) {
126-
int i;
127-
DBG_INF("success decompressing");
128-
for (i = 0 ; i < decompressed_size; i++) {
129-
if (i && (i % 30 == 0)) {
130-
printf("\n\t\t");
131-
}
132-
printf("%.2X ", (int)*((char*)&(decompressed_data[i])));
133-
DBG_INF_FMT("%.2X ", (int)*((char*)&(decompressed_data[i])));
134-
}
135-
} else {
136-
DBG_INF("error decompressing");
137-
}
138-
mnd_free(decompressed_data);
139-
}
140-
#endif /* WHEN_WE_NEED_TO_CHECK_WHETHER_COMPRESSION_WORKS_CORRECTLY */
141166
} else
142167
#endif /* MYSQLND_COMPRESSION_ENABLED */
143168
{

0 commit comments

Comments
 (0)