Skip to content

Commit e63699f

Browse files
msimacekmsimacek
committed
[GR-68825] Fix truncated decompression in java zlib backend
PullRequest: graalpython/3960
2 parents 763a4bd + 464aca2 commit e63699f

File tree

12 files changed

+75
-48
lines changed

12 files changed

+75
-48
lines changed

‎graalpython/com.oracle.graal.python.shell/src/com/oracle/graal/python/shell/GraalPythonMain.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -182,6 +182,7 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
182182
origArgs = new ArrayList<>();
183183
boolean posixBackendSpecified = false;
184184
boolean sha3BackendSpecified = false;
185+
boolean compressionBackendSpecified = false;
185186
boolean installSignalHandlersSpecified = false;
186187
boolean isolateNativeModulesSpecified = false;
187188
for (Iterator<String> argumentIterator = arguments.iterator(); argumentIterator.hasNext();) {
@@ -271,7 +272,8 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
271272
matchesPythonOption(arg, "StdLibHome") ||
272273
matchesPythonOption(arg, "CAPI") ||
273274
matchesPythonOption(arg, "PosixModuleBackend") ||
274-
matchesPythonOption(arg, "Sha3ModuleBackend")) {
275+
matchesPythonOption(arg, "Sha3ModuleBackend") ||
276+
matchesPythonOption(arg, "CompressionModulesBackend")) {
275277
addRelaunchArg(arg);
276278
}
277279
if (matchesPythonOption(arg, "PosixModuleBackend")) {
@@ -280,6 +282,9 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
280282
if (matchesPythonOption(arg, "Sha3ModuleBackend")) {
281283
sha3BackendSpecified = true;
282284
}
285+
if (matchesPythonOption(arg, "CompressionModulesBackend")) {
286+
compressionBackendSpecified = true;
287+
}
283288
if (matchesPythonOption(arg, "InstallSignalHandlers")) {
284289
installSignalHandlersSpecified = true;
285290
}
@@ -444,6 +449,9 @@ protected List<String> preprocessArguments(List<String> givenArgs, Map<String, S
444449
if (!sha3BackendSpecified) {
445450
polyglotOptions.put("python.Sha3ModuleBackend", "native");
446451
}
452+
if (!compressionBackendSpecified) {
453+
polyglotOptions.put("python.CompressionModulesBackend", "native");
454+
}
447455
if (!installSignalHandlersSpecified) {
448456
polyglotOptions.put("python.InstallSignalHandlers", "true");
449457
}

‎graalpython/com.oracle.graal.python.test/src/tests/test_zlib.py‎

Lines changed: 26 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -2,23 +2,12 @@
22
# Copyright (C) 1996-2017 Python Software Foundation
33
#
44
# Licensed under the PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
5-
6-
try:
7-
__graalpython__.zlib_module_backend()
8-
except:
9-
class GP:
10-
def zlib_module_backend(self):
11-
return 'cpython'
12-
13-
def _disable_native_zlib(self, flag):
14-
return None
15-
__graalpython__ = GP()
16-
5+
import binascii
176
import os
7+
import random
8+
import sys
189
import unittest
1910
import zlib
20-
import binascii
21-
import sys
2211

2312
pintNumber = 98765432109876543210
2413
longNumber = 9876543210
@@ -257,8 +246,6 @@ def test_GR65704():
257246
contents = b"The quick brown fox jumped over the lazy dog"
258247
wbits = 27
259248

260-
__graalpython__._disable_native_zlib(True)
261-
262249
compressed = zlib.compress(contents, wbits=wbits)
263250
decompressor = zlib.decompressobj(wbits=wbits)
264251

@@ -268,6 +255,28 @@ def test_GR65704():
268255
decompressed += out
269256
decompressed += decompressor.flush()
270257

271-
__graalpython__._disable_native_zlib(False)
258+
assert decompressed == contents
259+
260+
def test_large_chunk():
261+
contents = random.randbytes(5000)
262+
wbits = 31
263+
264+
compressed = zlib.compress(contents, wbits=wbits)
265+
decompressor = zlib.decompressobj(wbits=wbits)
266+
267+
decompressed = decompressor.decompress(compressed)
268+
269+
assert decompressed == contents
270+
271+
def test_various_chunks():
272+
contents = random.randbytes(5000)
273+
wbits = 31
274+
275+
compressed = zlib.compress(contents, wbits=wbits)
276+
decompressor = zlib.decompressobj(wbits=wbits)
277+
278+
decompressed = decompressor.decompress(compressed[:10])
279+
decompressed += decompressor.decompress(compressed[10:200])
280+
decompressed += decompressor.decompress(compressed[200:])
272281

273282
assert decompressed == contents

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java‎

Lines changed: 0 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1263,18 +1263,4 @@ static Object create(VirtualFrame frame, Object cls,
12631263
return objectNode.execute(frame, cls, PythonUtils.EMPTY_OBJECT_ARRAY, PKeyword.EMPTY_KEYWORDS);
12641264
}
12651265
}
1266-
1267-
@Builtin(name = "_disable_native_zlib", minNumOfPositionalArgs = 1)
1268-
@GenerateNodeFactory
1269-
abstract static class DisableNativeZlibNode extends PythonUnaryBuiltinNode {
1270-
@Specialization
1271-
Object disableNativeZlib(boolean disable) {
1272-
if (disable) {
1273-
getContext().getNFIZlibSupport().notAvailable();
1274-
} else {
1275-
getContext().getNFIZlibSupport().setAvailable();
1276-
}
1277-
return PNone.NONE;
1278-
}
1279-
}
12801266
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/zlib/JavaDecompress.java‎

Lines changed: 21 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747

4848
import java.io.ByteArrayInputStream;
4949
import java.io.ByteArrayOutputStream;
50+
import java.io.EOFException;
5051
import java.io.IOException;
5152
import java.io.InputStream;
5253
import java.util.zip.DataFormatException;
@@ -112,7 +113,7 @@ public Inflater getInflater() {
112113
return inf;
113114
}
114115

115-
public void setInput() throws IOException {
116+
public void fillInput() throws IOException {
116117
fill();
117118
}
118119
}
@@ -162,7 +163,7 @@ private static boolean isGZIPStreamReady(DecompressStream stream, byte[] data, i
162163
// GZIPInputStream will read the header during initialization
163164
stream.stream = new GZIPDecompressStream(stream.in);
164165
stream.inflater = stream.stream.getInflater();
165-
stream.stream.setInput();
166+
stream.stream.fillInput();
166167
return true;
167168
}
168169
} catch (ZipException ze) {
@@ -178,7 +179,7 @@ private static boolean isGZIPStreamFinishing(DecompressStream stream, byte[] dat
178179
stream.in.append(data, 0, length);
179180
try {
180181
if (stream.in.length() >= HEADER_TRAILER_SIZE) {
181-
stream.stream.setInput();
182+
stream.stream.fillInput();
182183
// this should trigger reading trailer
183184
stream.stream.read();
184185
stream.stream = null;
@@ -243,22 +244,33 @@ private byte[] createByteArray(byte[] bytes, int length, int maxLength, int bufS
243244
return EMPTY_BYTE_ARRAY;
244245
}
245246

246-
int maxLen = maxLength == 0 ? Integer.MAX_VALUE : maxLength;
247+
int maxLen = maxLength <= 0 ? Integer.MAX_VALUE : maxLength;
247248
byte[] result = new byte[Math.min(maxLen, bufSize)];
248249

249-
int bytesWritten = result.length;
250250
ByteArrayOutputStream baos = new ByteArrayOutputStream();
251251
boolean zdictIsSet = false;
252-
while (baos.size() < maxLen && bytesWritten == result.length) {
252+
while (baos.size() < maxLen && !stream.inflater.finished()) {
253+
if (stream.inflater.needsInput()) {
254+
if (stream.stream == null) {
255+
break;
256+
}
257+
try {
258+
stream.stream.fillInput();
259+
} catch (EOFException e) {
260+
break;
261+
} catch (IOException e) {
262+
throw CompilerDirectives.shouldNotReachHere(e);
263+
}
264+
}
265+
int bytesWritten;
253266
try {
254267
int len = Math.min(maxLen - baos.size(), result.length);
255268
bytesWritten = stream.inflater.inflate(result, 0, len);
256269
if (bytesWritten == 0 && !zdictIsSet && stream.inflater.needsDictionary()) {
257270
if (getZdict().length > 0) {
258271
setDictionary();
259272
zdictIsSet = true;
260-
// we inflate again with a dictionary
261-
bytesWritten = stream.inflater.inflate(result, 0, len);
273+
continue;
262274
} else {
263275
throw PRaiseNode.raiseStatic(nodeForRaise, ZLibError, WHILE_SETTING_ZDICT);
264276
}
@@ -320,7 +332,7 @@ protected static byte[] decompress(byte[] bytes, int length, int wbits, int bufs
320332
private void saveUnconsumedInput(byte[] data, int length,
321333
byte[] unusedDataBytes, int unconsumedTailLen, Node inliningTarget) {
322334
int unusedLen = getRemaining();
323-
byte[] tail = PythonUtils.arrayCopyOfRange(data, length - unusedLen, length);
335+
byte[] tail = PythonUtils.arrayCopyOfRange(data, Math.max(0, length - unusedLen), length);
324336
PythonLanguage language = PythonLanguage.get(inliningTarget);
325337
if (isEof()) {
326338
if (unconsumedTailLen > 0) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIBz2Support.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ public String signature() {
153153
@CompilerDirectives.CompilationFinal private boolean available;
154154

155155
private NFIBz2Support(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) {
156-
if (context.isNativeAccessAllowed()) {
156+
if (context.useNativeCompressionModules()) {
157157
this.pythonContext = context;
158158
this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), Bz2NativeFunctions.values(),
159159
backend, noNativeAccessHelp, false);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFILZMASupport.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,7 @@ public String signature() {
293293
@CompilerDirectives.CompilationFinal private boolean available;
294294

295295
private NFILZMASupport(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) {
296-
if (context.isNativeAccessAllowed()) {
296+
if (context.useNativeCompressionModules()) {
297297
this.pythonContext = context;
298298
this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME), LZMANativeFunctions.values(),
299299
backend, noNativeAccessHelp, true);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/NFIZlibSupport.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,7 @@ public String signature() {
269269
@CompilerDirectives.CompilationFinal private boolean available;
270270

271271
private NFIZlibSupport(PythonContext context, NativeLibrary.NFIBackend backend, String noNativeAccessHelp) {
272-
if (context.isNativeAccessAllowed()) {
272+
if (context.useNativeCompressionModules()) {
273273
this.pythonContext = context;
274274
this.typedNativeLib = NativeLibrary.create(PythonContext.getSupportLibName(SUPPORTING_NATIVE_LIB_NAME),
275275
ZlibNativeFunctions.values(), backend, noNativeAccessHelp, true);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonContext.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2337,6 +2337,15 @@ public boolean isExecutableAccessAllowed() {
23372337
return getEnv().isHostLookupAllowed() || isNativeAccessAllowed();
23382338
}
23392339

2340+
public boolean useNativeCompressionModules() {
2341+
if (isNativeAccessAllowed()) {
2342+
TruffleString option = getLanguage().getEngineOption(PythonOptions.CompressionModulesBackend);
2343+
TruffleString.EqualNode eqNode = TruffleString.EqualNode.getUncached();
2344+
return !eqNode.execute(T_JAVA, option, TS_ENCODING);
2345+
}
2346+
return false;
2347+
}
2348+
23402349
/**
23412350
* Trigger any pending asynchronous actions
23422351
*/

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/runtime/PythonOptions.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -195,12 +195,15 @@ public static void checkBytecodeDSLEnv() {
195195
}
196196
}));
197197

198-
@EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the POSIX module.", usageSyntax = "java|native|llvm", stability = OptionStability.STABLE) //
198+
@EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the POSIX module.", usageSyntax = "java|native", stability = OptionStability.STABLE) //
199199
public static final OptionKey<TruffleString> PosixModuleBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE);
200200

201201
@EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the Sha3 module.", usageSyntax = "java|native", stability = OptionStability.STABLE) //
202202
public static final OptionKey<TruffleString> Sha3ModuleBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE);
203203

204+
@EngineOption @Option(category = OptionCategory.USER, help = "Choose the backend for the Zlib, Bz2, and LZMA modules.", usageSyntax = "java|native", stability = OptionStability.STABLE) //
205+
public static final OptionKey<TruffleString> CompressionModulesBackend = new OptionKey<>(T_JAVA, TS_OPTION_TYPE);
206+
204207
@Option(category = OptionCategory.USER, help = "Install default signal handlers on startup", usageSyntax = "true|false", stability = OptionStability.STABLE) //
205208
public static final OptionKey<Boolean> InstallSignalHandlers = new OptionKey<>(false);
206209

‎graalpython/lib-python/3/test/test_zlib.py‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,6 @@
1515
class GP:
1616
def zlib_module_backend(self):
1717
return 'cpython'
18-
19-
def _disable_native_zlib(self):
20-
return None
2118
__graalpython__ = GP()
2219

2320
zlib = import_helper.import_module('zlib')

0 commit comments

Comments
 (0)