Skip to content

Commit da8f93f

Browse files
steve-ssteve-s
committed
Add test of multi-threaded import of Python packages
1 parent a2241c1 commit da8f93f

File tree

5 files changed

+311
-1
lines changed

5 files changed

+311
-1
lines changed

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/cext/test/MultithreadedImportTestBase.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/*
2+
* Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.cext.test;
42+
43+
import java.lang.management.ManagementFactory;
44+
import java.lang.management.ThreadInfo;
45+
import java.lang.management.ThreadMXBean;
46+
import java.util.ArrayList;
47+
import java.util.concurrent.ExecutorService;
48+
import java.util.concurrent.Executors;
49+
import java.util.concurrent.Future;
50+
import java.util.concurrent.TimeUnit;
51+
import java.util.concurrent.TimeoutException;
52+
import java.util.stream.Collectors;
53+
54+
import org.graalvm.polyglot.Context;
55+
import org.junit.Assert;
56+
57+
public class MultithreadedImportTestBase {
58+
private static final int ITERATIONS_LIMIT = 10;
59+
60+
// This test should be executed in its own process. It tests that there are no deadlocks. We do
61+
// not want to wait for the gate job timeout and timeout the test ourselves after shorter period
62+
// of time, but for that we must not call ExecutorService#close, which would be waiting for the
63+
// threads to finish
64+
@SuppressWarnings("resource")
65+
static void multithreadedImportTest(int numberOfThreads, Context context) {
66+
try {
67+
ExecutorService executor = Executors.newFixedThreadPool(numberOfThreads);
68+
var tasks = new ArrayList<Future<?>>();
69+
for (String pkg : PACKAGES.trim().split("\n")) {
70+
log("Starting import: %s", pkg);
71+
tasks.add(executor.submit(() -> {
72+
log("Importing %s on thread %s", pkg, Thread.currentThread());
73+
context.eval("python", "import " + pkg);
74+
}));
75+
}
76+
77+
int iteration = 0;
78+
while (!tasks.isEmpty() && iteration++ < ITERATIONS_LIMIT) {
79+
log("Iteration %s, looping over remaining %d unfinished tasks", iteration, tasks.size());
80+
var finishedTasks = tasks.stream().filter(task -> {
81+
try {
82+
task.get(1000, TimeUnit.MILLISECONDS);
83+
return true;
84+
} catch (TimeoutException timeoutEx) {
85+
return false;
86+
} catch (Exception ex) {
87+
log("Caught exception: %s", ex);
88+
throw new RuntimeException(ex);
89+
}
90+
}).collect(Collectors.toCollection(ArrayList::new));
91+
tasks.removeAll(finishedTasks);
92+
}
93+
94+
if (tasks.isEmpty()) {
95+
executor.shutdown();
96+
} else {
97+
// otherwise do not wait for the threads to finish, just dump them and continue to
98+
// fail the assertion below
99+
try {
100+
System.out.println("There are unfinished tasks. This failure is inherently transient. " +
101+
"Please report any failure. Thread dump is below if available:");
102+
ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean();
103+
for (ThreadInfo threadInfo : threadMxBean.dumpAllThreads(true, true, 20)) {
104+
System.out.print(threadInfo.toString());
105+
}
106+
} catch (UnsupportedOperationException ignored) {
107+
}
108+
}
109+
110+
Assert.assertTrue("Unfinished tasks", tasks.isEmpty());
111+
log("DONE: %s", MultithreadedImportTestBase.class.getSimpleName());
112+
} finally {
113+
context.close(true);
114+
}
115+
}
116+
117+
private static void log(String fmt, Object... args) {
118+
System.out.printf(fmt + "%n", args);
119+
}
120+
121+
private static final String PACKAGES = """
122+
csv
123+
configparser
124+
tomllib
125+
hashlib
126+
os
127+
_testcapi
128+
io
129+
time
130+
logging
131+
ctypes
132+
argparse
133+
_sqlite3
134+
_cpython_sre
135+
threading
136+
multiprocessing
137+
sched
138+
contextvars
139+
json
140+
pyexpat
141+
base64
142+
html
143+
locale
144+
shlex
145+
venv
146+
ast
147+
re
148+
difflib
149+
zlib
150+
""";
151+
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/cext/test/MultithreadedImportTestJava.java‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
/*
2+
* Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.cext.test;
42+
43+
import static com.oracle.graal.python.cext.test.MultithreadedImportTestBase.multithreadedImportTest;
44+
45+
import org.graalvm.polyglot.Context;
46+
import org.junit.Test;
47+
48+
public class MultithreadedImportTestJava {
49+
@Test
50+
public void testImportOnMultipleThreads() {
51+
Context context = Context.newBuilder().allowAllAccess(true).option("python.PosixModuleBackend", "java").build();
52+
multithreadedImportTest(8, context);
53+
}
54+
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/cext/test/MultithreadedImportTestNative.java‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
/*
2+
* Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.cext.test;
42+
43+
import org.graalvm.polyglot.Context;
44+
import org.junit.Test;
45+
46+
public class MultithreadedImportTestNative extends MultithreadedImportTestBase {
47+
@Test
48+
public void testImportOnMultipleThreads() {
49+
Context context = Context.newBuilder().allowAllAccess(true).option("python.PosixModuleBackend", "native").build();
50+
multithreadedImportTest(32, context);
51+
}
52+
}

‎graalpython/com.oracle.graal.python.test/src/com/oracle/graal/python/cext/test/package-info.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
/*
2+
* Copyright (c) 2025, 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
42+
/**
43+
* Package for tests that need to run in a separate process due to C extensions usage. Add a
44+
* dedicated TestConfig into mx_graalpython#punittest for each test in this package.
45+
*/
46+
package com.oracle.graal.python.cext.test;

‎mx.graalpython/mx_graalpython.py‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -596,6 +596,8 @@ class TestConfig:
596596
reportConfig: Union[Task, bool, None] = report
597597
def __str__(self):
598598
return f"args={self.args!r}, useResources={self.useResources}, report={self.reportConfig}"
599+
def __post_init__(self):
600+
assert ' ' not in self.identifier
599601

600602
configs = []
601603
skip_leak_tests = False
@@ -616,6 +618,11 @@ def __str__(self):
616618
configs += [
617619
TestConfig("junit", vm_args + graalpy_tests + args, True),
618620
TestConfig("junit", vm_args + graalpy_tests + args, False),
621+
# Tests that must run in their own process due to C extensions usage
622+
TestConfig("multi-threaded-import-java", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestNative'] + args, True),
623+
TestConfig("multi-threaded-import-java", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestNative'] + args, False),
624+
TestConfig("multi-threaded-import-native", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestJava'] + args, True),
625+
TestConfig("multi-threaded-import-native", vm_args + ['com.oracle.graal.python.cext.test.MultithreadedImportTestJava'] + args, False),
619626
]
620627

621628
if '--regex' not in args:
@@ -1234,7 +1241,7 @@ def graalpython_gate_runner(_, tasks):
12341241
"--verbose",
12351242
"--no-leak-tests",
12361243
"--regex",
1237-
r'((graal\.python\.test\.integration)|(graal\.python\.test\.(builtin|interop|util))|(org\.graalvm\.python\.embedding\.(test|test\.integration)))'
1244+
r'((graal\.python\.test\.integration)|(graal\.python\.test\.(builtin|interop|util))|(graal\.python\.cext\.test))'
12381245
],
12391246
report=True
12401247
)

0 commit comments

Comments
 (0)