Skip to content

Commit c4d46bc

Browse files
steve-ssteve-s
committed
Bytecode DSL: use custom yield operation to merge generator function and generator body into one RootNode; several related to generators
1 parent 9309245 commit c4d46bc

File tree

20 files changed

+349
-344
lines changed

20 files changed

+349
-344
lines changed

‎graalpython/com.oracle.graal.python.test/src/tests/test_code.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
# Copyright (c) 2018, 2024, Oracle and/or its affiliates. All rights reserved.
1+
# Copyright (c) 2018, 2025, Oracle and/or its affiliates. All rights reserved.
22
# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
33
#
44
# The Universal Permissive License (UPL), Version 1.0
@@ -184,7 +184,7 @@ def inner():
184184
assert "fn doc" not in code.co_consts
185185
for const in code.co_consts:
186186
if type(const) == types.CodeType:
187-
code = const
187+
code = const
188188
assert "fn doc" in code.co_consts
189189
assert "this is fun" not in code.co_consts
190190
for const in code.co_consts:
@@ -209,7 +209,7 @@ def inner():
209209
assert "gen doc" not in code.co_consts
210210
for const in code.co_consts:
211211
if type(const) == types.CodeType:
212-
code = const
212+
code = const
213213
assert "gen doc" in code.co_consts
214214
assert "this is fun" not in code.co_consts
215215
for const in code.co_consts:
@@ -223,3 +223,11 @@ def test_consts_do_not_leak_java_types():
223223
code = compile(codestr, '<test>', 'exec')
224224
for const in code.co_consts:
225225
assert isinstance(const, (str, tuple)) or const is None
226+
227+
228+
def test_generator_and_gen_body_code_are_equal():
229+
def g():
230+
yield 42
231+
232+
x = g()
233+
assert g.__code__ is x.gi_code

‎graalpython/com.oracle.graal.python.test/src/tests/unittest_tags_bytecode_dsl/test_generators.txt‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ test.test_generators.FinalizationTest.test_lambda_generator @ linux-x86_64
2121
test.test_generators.GeneratorStackTraceTest.test_send_with_yield_from @ linux-x86_64
2222
test.test_generators.GeneratorStackTraceTest.test_throw_with_yield_from @ linux-x86_64
2323
test.test_generators.GeneratorTest.test_copy @ linux-x86_64
24+
test.test_generators.GeneratorTest.test_cr_frame_f_back @ linux-x86_64
25+
test.test_generators.GeneratorTest.test_gi_frame_f_back @ linux-x86_64
2426
test.test_generators.GeneratorTest.test_issue103488 @ linux-x86_64
2527
test.test_generators.GeneratorTest.test_name @ linux-x86_64
2628
test.test_generators.GeneratorTest.test_pickle @ linux-x86_64
@@ -29,4 +31,4 @@ test.test_generators.GeneratorThrowTest.test_exception_context_with_yield @ linu
2931
test.test_generators.GeneratorThrowTest.test_exception_context_with_yield_from @ linux-x86_64
3032
test.test_generators.GeneratorThrowTest.test_exception_context_with_yield_from_with_context_cycle @ linux-x86_64
3133
test.test_generators.GeneratorThrowTest.test_exception_context_with_yield_inside_generator @ linux-x86_64
32-
test.test_generators.GeneratorThrowTest.test_throw_after_none_exc_type @ linux-x86_64
34+
test.test_generators.GeneratorThrowTest.test_throw_after_none_exc_type @ linux-x86_64

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/modules/GraalPythonModuleBuiltins.java‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,6 @@
163163
import com.oracle.graal.python.runtime.sequence.storage.NativeSequenceStorage;
164164
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
165165
import com.oracle.graal.python.util.PythonUtils;
166-
import com.oracle.truffle.api.CallTarget;
167166
import com.oracle.truffle.api.CompilerAsserts;
168167
import com.oracle.truffle.api.CompilerDirectives;
169168
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
@@ -361,15 +360,15 @@ private void runFile(PythonContext context, TruffleString inputFilePath) {
361360
throw new PythonExitException(this, 2);
362361
}
363362
PythonLanguage language = context.getLanguage();
364-
CallTarget callTarget = context.getEnv().parsePublic(source);
363+
RootCallTarget callTarget = (RootCallTarget) context.getEnv().parsePublic(source);
365364
Object[] arguments = PArguments.create();
366365
PythonModule mainModule = context.getMainModule();
367366
PDict mainDict = GetOrCreateDictNode.executeUncached(mainModule);
368367
PArguments.setGlobals(arguments, mainModule);
369368
PArguments.setSpecialArgument(arguments, mainDict);
370369
PArguments.setException(arguments, PException.NO_EXCEPTION);
371370
context.initializeMainModule(inputFilePath);
372-
Object state = ExecutionContext.IndirectCalleeContext.enterIndirect(language, context, arguments);
371+
Object state = ExecutionContext.IndirectCalleeContext.enterIndirect(language, context, arguments, callTarget);
373372
try {
374373
callTarget.call(arguments);
375374
} finally {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,6 @@
5656
import com.oracle.graal.python.nodes.bytecode.PBytecodeGeneratorFunctionRootNode;
5757
import com.oracle.graal.python.nodes.bytecode.PBytecodeRootNode;
5858
import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit;
59-
import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLGeneratorFunctionRootNode;
60-
import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode;
6159
import com.oracle.graal.python.nodes.util.BadOPCodeNode;
6260
import com.oracle.graal.python.runtime.ExecutionContext.IndirectCallContext;
6361
import com.oracle.graal.python.runtime.IndirectCallData;
@@ -163,9 +161,6 @@ private RootCallTarget deserializeForBytecodeInterpreter(PythonLanguage language
163161
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
164162
BytecodeDSLCodeUnit code = (BytecodeDSLCodeUnit) codeUnit;
165163
rootNode = code.createRootNode(context, PythonUtils.createFakeSource());
166-
if (code.isGeneratorOrCoroutine()) {
167-
rootNode = new PBytecodeDSLGeneratorFunctionRootNode(language, rootNode.getFrameDescriptor(), (PBytecodeDSLRootNode) rootNode, code.name);
168-
}
169164
} else {
170165
BytecodeCodeUnit code = (BytecodeCodeUnit) codeUnit;
171166
if (cellvars != null && !Arrays.equals(code.cellvars, cellvars) || freevars != null && !Arrays.equals(code.freevars, freevars)) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import com.oracle.graal.python.builtins.objects.bytes.PBytes;
5858
import com.oracle.graal.python.builtins.objects.ellipsis.PEllipsis;
5959
import com.oracle.graal.python.builtins.objects.function.Signature;
60+
import com.oracle.graal.python.builtins.objects.generator.PGenerator;
6061
import com.oracle.graal.python.builtins.objects.object.PythonBuiltinObject;
6162
import com.oracle.graal.python.builtins.objects.tuple.PTuple;
6263
import com.oracle.graal.python.compiler.BytecodeCodeUnit;
@@ -67,7 +68,6 @@
6768
import com.oracle.graal.python.nodes.bytecode.PBytecodeGeneratorRootNode;
6869
import com.oracle.graal.python.nodes.bytecode.PBytecodeRootNode;
6970
import com.oracle.graal.python.nodes.bytecode_dsl.BytecodeDSLCodeUnit;
70-
import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLGeneratorFunctionRootNode;
7171
import com.oracle.graal.python.nodes.bytecode_dsl.PBytecodeDSLRootNode;
7272
import com.oracle.graal.python.nodes.object.IsForeignObjectNode;
7373
import com.oracle.graal.python.runtime.GilNode;
@@ -82,7 +82,6 @@
8282
import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary;
8383
import com.oracle.truffle.api.RootCallTarget;
8484
import com.oracle.truffle.api.bytecode.BytecodeNode;
85-
import com.oracle.truffle.api.bytecode.ContinuationRootNode;
8685
import com.oracle.truffle.api.dsl.Cached;
8786
import com.oracle.truffle.api.dsl.Cached.Exclusive;
8887
import com.oracle.truffle.api.interop.InteropLibrary;
@@ -368,21 +367,16 @@ private static TruffleString[] extractNames(RootNode node) {
368367

369368
private static RootNode rootNodeForExtraction(RootNode rootNode) {
370369
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
371-
if (rootNode instanceof PBytecodeDSLGeneratorFunctionRootNode generatorFunctionRootNode) {
372-
return generatorFunctionRootNode.getBytecodeRootNode();
373-
}
374-
if (rootNode instanceof ContinuationRootNode generatorRootNode) {
375-
return (RootNode) generatorRootNode.getSourceRootNode();
376-
}
370+
return PGenerator.unwrapContinuationRoot(rootNode);
377371
} else {
378372
if (rootNode instanceof PBytecodeGeneratorFunctionRootNode generatorFunctionRootNode) {
379373
return generatorFunctionRootNode.getBytecodeRootNode();
380374
}
381375
if (rootNode instanceof PBytecodeGeneratorRootNode generatorRootNode) {
382376
return generatorRootNode.getBytecodeRootNode();
383377
}
378+
return rootNode;
384379
}
385-
return rootNode;
386380
}
387381

388382
@TruffleBoundary

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/CommonGeneratorBuiltins.java‎

Lines changed: 5 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,6 @@
6161
import com.oracle.graal.python.builtins.objects.exception.PBaseException;
6262
import com.oracle.graal.python.builtins.objects.exception.PrepareExceptionNode;
6363
import com.oracle.graal.python.builtins.objects.frame.PFrame;
64-
import com.oracle.graal.python.builtins.objects.function.PArguments;
6564
import com.oracle.graal.python.builtins.objects.traceback.PTraceback;
6665
import com.oracle.graal.python.builtins.objects.type.slots.TpSlotIterNext.TpIterNextBuiltin;
6766
import com.oracle.graal.python.lib.IteratorExhausted;
@@ -81,7 +80,6 @@
8180
import com.oracle.graal.python.runtime.PythonOptions;
8281
import com.oracle.graal.python.runtime.exception.PException;
8382
import com.oracle.graal.python.runtime.object.PFactory;
84-
import com.oracle.truffle.api.Truffle;
8583
import com.oracle.truffle.api.bytecode.ContinuationResult;
8684
import com.oracle.truffle.api.dsl.Bind;
8785
import com.oracle.truffle.api.dsl.Cached;
@@ -156,23 +154,14 @@ static Object cached(VirtualFrame frame, Node inliningTarget, PGenerator self, O
156154
static Object cachedBytecodeDSL(VirtualFrame frame, Node inliningTarget, PGenerator self, Object sendValue,
157155
@Cached(parameters = "self.getCurrentCallTarget()") DirectCallNode callNode,
158156
@Exclusive @Cached CallDispatchers.SimpleDirectInvokeNode invoke,
159-
@Cached("self.getContinuation() == null") boolean firstCall,
160157
@Exclusive @Cached InlinedBranchProfile returnProfile,
161158
@Exclusive @Cached IsBuiltinObjectProfile errorProfile,
162159
@Exclusive @Cached PRaiseNode raiseNode) {
163160
self.setRunning(true);
164161
Object generatorResult;
165162
try {
166-
ContinuationResult continuation = self.getContinuation();
167-
Object[] arguments;
168-
if (firstCall) {
169-
// First invocation: call the regular root node.
170-
arguments = self.getCallArguments(sendValue);
171-
} else {
172-
// Subsequent invocations: call a continuation root node.
173-
self.prepareResume();
174-
arguments = new Object[]{continuation.getFrame(), sendValue};
175-
}
163+
self.prepareResume();
164+
Object[] arguments = new Object[]{self.getGeneratorFrame(), sendValue};
176165
generatorResult = invoke.execute(frame, inliningTarget, callNode, arguments);
177166
} catch (PException e) {
178167
throw handleException(self, inliningTarget, errorProfile, raiseNode, e);
@@ -215,24 +204,14 @@ static Object generic(VirtualFrame frame, Node inliningTarget, PGenerator self,
215204
@Megamorphic
216205
static Object genericBytecodeDSL(VirtualFrame frame, Node inliningTarget, PGenerator self, Object sendValue,
217206
@Exclusive @Cached CallDispatchers.SimpleIndirectInvokeNode invoke,
218-
@Exclusive @Cached InlinedConditionProfile firstInvocationProfile,
219207
@Exclusive @Cached InlinedBranchProfile returnProfile,
220208
@Exclusive @Cached IsBuiltinObjectProfile errorProfile,
221209
@Exclusive @Cached PRaiseNode raiseNode) {
222210
self.setRunning(true);
223211
Object generatorResult;
224212
try {
225-
ContinuationResult continuation = self.getContinuation();
226-
Object[] arguments;
227-
if (firstInvocationProfile.profile(inliningTarget, continuation == null)) {
228-
// First invocation: call the regular root node.
229-
arguments = self.getCallArguments(sendValue);
230-
} else {
231-
// Subsequent invocations: call a continuation root node.
232-
self.prepareResume();
233-
arguments = new Object[]{continuation.getFrame(), sendValue};
234-
}
235-
213+
self.prepareResume();
214+
Object[] arguments = new Object[]{self.getGeneratorFrame(), sendValue};
236215
generatorResult = invoke.execute(frame, inliningTarget, self.getCurrentCallTarget(), arguments);
237216
} catch (PException e) {
238217
throw handleException(self, inliningTarget, errorProfile, raiseNode, e);
@@ -359,12 +338,7 @@ static Object sendThrow(VirtualFrame frame, PGenerator self, Object typ, Object
359338
// its frame to the traceback manually.
360339
self.markAsFinished();
361340
Node location = self.getCurrentCallTarget().getRootNode();
362-
MaterializedFrame generatorFrame;
363-
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
364-
generatorFrame = Truffle.getRuntime().createMaterializedFrame(PArguments.create(), self.getRootNode().getFrameDescriptor());
365-
} else {
366-
generatorFrame = self.getGeneratorFrame();
367-
}
341+
MaterializedFrame generatorFrame = self.getGeneratorFrame();
368342
PFrame pFrame = MaterializeFrameNode.materializeGeneratorFrame(location, generatorFrame, PFrame.Reference.EMPTY);
369343
FrameInfo info = (FrameInfo) generatorFrame.getFrameDescriptor().getInfo();
370344
pFrame.setLine(info.getFirstLineNumber());

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/generator/GeneratorBuiltins.java‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,6 @@
6060
import com.oracle.graal.python.runtime.PythonOptions;
6161
import com.oracle.graal.python.runtime.object.PFactory;
6262
import com.oracle.truffle.api.bytecode.BytecodeLocation;
63-
import com.oracle.truffle.api.bytecode.ContinuationResult;
6463
import com.oracle.truffle.api.dsl.Bind;
6564
import com.oracle.truffle.api.dsl.Cached;
6665
import com.oracle.truffle.api.dsl.GenerateNodeFactory;
@@ -190,8 +189,7 @@ static Object getFrame(PGenerator self) {
190189
return PNone.NONE;
191190
} else {
192191
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
193-
ContinuationResult continuation = self.getContinuation();
194-
BytecodeLocation location = continuation != null ? continuation.getBytecodeLocation() : null;
192+
BytecodeLocation location = self.getCurrentRootNode().getLocation();
195193
MaterializedFrame generatorFrame = self.getGeneratorFrame();
196194
BytecodeDSLFrameInfo info = (BytecodeDSLFrameInfo) generatorFrame.getFrameDescriptor().getInfo();
197195
PFrame frame = MaterializeFrameNode.materializeGeneratorFrame(location != null ? location.getBytecodeNode() : null, generatorFrame, PFrame.Reference.EMPTY);

0 commit comments

Comments
 (0)