Skip to content

Commit 0cffc0d

Browse files
msimacekmsimacek
committed
[GR-69728] Lazily load text sources
PullRequest: graalpython/4007
2 parents e62ab2d + e1520c1 commit 0cffc0d

File tree

7 files changed

+119
-46
lines changed

7 files changed

+119
-46
lines changed

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/PythonLanguage.java‎

Lines changed: 21 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,7 @@
109109
import com.oracle.graal.python.runtime.exception.PException;
110110
import com.oracle.graal.python.runtime.object.PFactory;
111111
import com.oracle.graal.python.util.Function;
112+
import com.oracle.graal.python.util.LazySource;
112113
import com.oracle.graal.python.util.PythonUtils;
113114
import com.oracle.graal.python.util.Supplier;
114115
import com.oracle.truffle.api.Assumption;
@@ -597,48 +598,48 @@ protected CallTarget parse(ParsingRequest request) {
597598

598599
public RootCallTarget callTargetFromBytecode(PythonContext context, Source source, CodeUnit code) {
599600
boolean internal = shouldMarkSourceInternal(context);
601+
SourceBuilder builder = null;
600602
// The original file path should be passed as the name
601603
String name = source.getName();
602604
if (name != null && !name.isEmpty()) {
603-
Source textSource = tryLoadSource(context, code, internal, name);
604-
if (textSource == null) {
605+
builder = sourceForOriginalFile(context, code, internal, name);
606+
if (builder == null) {
605607
if (name.startsWith(FROZEN_FILENAME_PREFIX) && name.endsWith(FROZEN_FILENAME_SUFFIX)) {
606608
String id = name.substring(FROZEN_FILENAME_PREFIX.length(), name.length() - FROZEN_FILENAME_SUFFIX.length());
607609
String fs = context.getEnv().getFileNameSeparator();
608610
String path = context.getStdlibHome() + fs + id.replace(".", fs) + J_PY_EXTENSION;
609-
textSource = tryLoadSource(context, code, internal, path);
610-
if (textSource == null) {
611+
builder = sourceForOriginalFile(context, code, internal, path);
612+
if (builder == null) {
611613
path = context.getStdlibHome() + fs + id.replace(".", fs) + fs + "__init__.py";
612-
textSource = tryLoadSource(context, code, internal, path);
614+
builder = sourceForOriginalFile(context, code, internal, path);
613615
}
614616
}
615617
}
616-
if (textSource != null) {
617-
source = textSource;
618-
}
619618
}
620-
if (internal && !source.isInternal()) {
621-
source = Source.newBuilder(source).internal(true).build();
619+
if (builder == null) {
620+
builder = Source.newBuilder(source).internal(internal).content(Source.CONTENT_NONE);
622621
}
623622
RootNode rootNode;
623+
LazySource lazySource = new LazySource(builder);
624624

625625
if (PythonOptions.ENABLE_BYTECODE_DSL_INTERPRETER) {
626-
if (source.hasBytes()) {
627-
// Force a character-based source so that source sections work as expected.
628-
source = Source.newBuilder(source).content(Source.CONTENT_NONE).build();
629-
}
630-
rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(context, source);
626+
// TODO lazily load source in bytecode DSL interpreter too
627+
rootNode = ((BytecodeDSLCodeUnit) code).createRootNode(context, lazySource.getSource());
631628
} else {
632-
rootNode = PBytecodeRootNode.create(this, (BytecodeCodeUnit) code, source);
629+
rootNode = PBytecodeRootNode.create(this, (BytecodeCodeUnit) code, lazySource, internal);
633630
}
634631

635632
return PythonUtils.getOrCreateCallTarget(rootNode);
636633
}
637634

638-
private static Source tryLoadSource(PythonContext context, CodeUnit code, boolean internal, String path) {
635+
private static SourceBuilder sourceForOriginalFile(PythonContext context, CodeUnit code, boolean internal, String path) {
639636
try {
640-
return Source.newBuilder(PythonLanguage.ID, context.getEnv().getPublicTruffleFile(path)).name(code.name.toJavaStringUncached()).internal(internal).build();
641-
} catch (IOException | SecurityException | UnsupportedOperationException | InvalidPathException e) {
637+
TruffleFile file = context.getEnv().getPublicTruffleFile(path);
638+
if (!file.isReadable()) {
639+
return null;
640+
}
641+
return Source.newBuilder(PythonLanguage.ID, file).name(code.name.toJavaStringUncached()).internal(internal);
642+
} catch (SecurityException | UnsupportedOperationException | InvalidPathException e) {
642643
return null;
643644
}
644645
}
@@ -724,7 +725,7 @@ private RootNode compileForBytecodeInterpreter(ModTy mod, Source source, int opt
724725
Compiler compiler = new Compiler(parserCallbacks);
725726
CompilationUnit cu = compiler.compile(mod, EnumSet.noneOf(Compiler.Flags.class), optimize, futureFeatures);
726727
BytecodeCodeUnit co = cu.assemble();
727-
return PBytecodeRootNode.create(this, co, source, parserCallbacks);
728+
return PBytecodeRootNode.create(this, co, new LazySource(source), source.isInternal(), parserCallbacks);
728729
}
729730

730731
private RootNode compileForBytecodeDSLInterpreter(PythonContext context, ModTy mod, Source source, int optimize,

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/CodeNodes.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import com.oracle.graal.python.runtime.PythonContext;
6565
import com.oracle.graal.python.runtime.PythonOptions;
6666
import com.oracle.graal.python.runtime.object.PFactory;
67+
import com.oracle.graal.python.util.LazySource;
6768
import com.oracle.graal.python.util.PythonUtils;
6869
import com.oracle.graal.python.util.Supplier;
6970
import com.oracle.truffle.api.CallTarget;
@@ -176,7 +177,7 @@ private RootCallTarget deserializeForBytecodeInterpreter(PythonLanguage language
176177
code.outputCanQuicken, code.variableShouldUnbox,
177178
code.generalizeInputsMap, code.generalizeVarsMap);
178179
}
179-
rootNode = PBytecodeRootNode.create(context.getLanguage(), code, PythonUtils.createFakeSource());
180+
rootNode = PBytecodeRootNode.create(context.getLanguage(), code, new LazySource(PythonUtils.createFakeSource()), false);
180181
if (code.isGeneratorOrCoroutine()) {
181182
rootNode = new PBytecodeGeneratorFunctionRootNode(context.getLanguage(), rootNode.getFrameDescriptor(), (PBytecodeRootNode) rootNode, code.name);
182183
}

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/code/PCode.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -560,7 +560,7 @@ private static Object convertConstantToPythonSpace(RootNode rootNode, Object o)
560560
return PFactory.createCode(language, root.getCallTarget(), root.getSignature(), code);
561561
} else {
562562
BytecodeCodeUnit code = (BytecodeCodeUnit) o;
563-
PBytecodeRootNode bytecodeRootNode = PBytecodeRootNode.create(language, code, getSourceSection(rootNode).getSource());
563+
PBytecodeRootNode bytecodeRootNode = PBytecodeRootNode.create(language, code, ((PBytecodeRootNode) rootNode).getLazySource(), rootNode.isInternal());
564564
return PFactory.createCode(language, bytecodeRootNode.getCallTarget(), bytecodeRootNode.getSignature(), code);
565565
}
566566
} else if (o instanceof BigInteger) {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/MakeFunctionNode.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@
5454
import com.oracle.graal.python.nodes.PNodeWithContext;
5555
import com.oracle.graal.python.nodes.attributes.WriteAttributeToPythonObjectNode;
5656
import com.oracle.graal.python.runtime.object.PFactory;
57+
import com.oracle.graal.python.util.LazySource;
5758
import com.oracle.truffle.api.Assumption;
5859
import com.oracle.truffle.api.CompilerDirectives;
5960
import com.oracle.truffle.api.CompilerDirectives.CompilationFinal;
@@ -63,7 +64,6 @@
6364
import com.oracle.truffle.api.dsl.Cached;
6465
import com.oracle.truffle.api.dsl.Specialization;
6566
import com.oracle.truffle.api.frame.VirtualFrame;
66-
import com.oracle.truffle.api.source.Source;
6767

6868
public abstract class MakeFunctionNode extends PNodeWithContext {
6969
private final RootCallTarget callTarget;
@@ -134,9 +134,9 @@ int makeFunction(VirtualFrame frame, Object globals, int initialStackTop, int fl
134134
return stackTop;
135135
}
136136

137-
public static MakeFunctionNode create(PythonLanguage language, BytecodeCodeUnit code, Source source) {
137+
public static MakeFunctionNode create(PythonLanguage language, BytecodeCodeUnit code, LazySource lazySource, boolean internal) {
138138
RootCallTarget callTarget;
139-
PBytecodeRootNode bytecodeRootNode = PBytecodeRootNode.create(language, code, source);
139+
PBytecodeRootNode bytecodeRootNode = PBytecodeRootNode.create(language, code, lazySource, internal);
140140
if (code.isGeneratorOrCoroutine()) {
141141
// TODO what should the frameDescriptor be? does it matter?
142142
callTarget = new PBytecodeGeneratorFunctionRootNode(language, bytecodeRootNode.getFrameDescriptor(), bytecodeRootNode, code.name).getCallTarget();

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/PBytecodeRootNode.java‎

Lines changed: 20 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -237,6 +237,7 @@
237237
import com.oracle.graal.python.runtime.sequence.storage.LongSequenceStorage;
238238
import com.oracle.graal.python.runtime.sequence.storage.ObjectSequenceStorage;
239239
import com.oracle.graal.python.runtime.sequence.storage.SequenceStorage;
240+
import com.oracle.graal.python.util.LazySource;
240241
import com.oracle.graal.python.util.PythonUtils;
241242
import com.oracle.truffle.api.Assumption;
242243
import com.oracle.truffle.api.CompilerAsserts;
@@ -549,7 +550,7 @@ public final class PBytecodeRootNode extends PRootNode implements BytecodeOSRNod
549550
final int classcellIndex;
550551

551552
private final BytecodeCodeUnit co;
552-
private final Source source;
553+
private final LazySource lazySource;
553554
private SourceSection sourceSection;
554555
// For deferred deprecation warnings
555556
private final ParserCallbacksImpl parserCallbacks;
@@ -674,15 +675,15 @@ private static FrameDescriptor makeFrameDescriptor(BytecodeCodeUnit co, FrameInf
674675
}
675676

676677
@TruffleBoundary
677-
public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit co, Source source) {
678-
return create(language, co, source, null);
678+
public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit co, LazySource lazySource, boolean internal) {
679+
return create(language, co, lazySource, internal, null);
679680
}
680681

681682
@TruffleBoundary
682-
public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit co, Source source, ParserCallbacksImpl parserCallbacks) {
683+
public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit co, LazySource lazySource, boolean internal, ParserCallbacksImpl parserCallbacks) {
683684
BytecodeFrameInfo frameInfo = new BytecodeFrameInfo();
684685
FrameDescriptor fd = makeFrameDescriptor(co, frameInfo);
685-
PBytecodeRootNode rootNode = new PBytecodeRootNode(language, fd, co.computeSignature(), co, source, parserCallbacks);
686+
PBytecodeRootNode rootNode = new PBytecodeRootNode(language, fd, co.computeSignature(), co, lazySource, internal, parserCallbacks);
686687
PythonContext context = PythonContext.get(rootNode);
687688
if (context != null && context.getOption(PythonOptions.EagerlyMaterializeInstrumentationNodes)) {
688689
rootNode.adoptChildren();
@@ -693,15 +694,15 @@ public static PBytecodeRootNode create(PythonLanguage language, BytecodeCodeUnit
693694
}
694695

695696
@TruffleBoundary
696-
private PBytecodeRootNode(PythonLanguage language, FrameDescriptor fd, Signature sign, BytecodeCodeUnit co, Source source, ParserCallbacksImpl parserCallbacks) {
697+
private PBytecodeRootNode(PythonLanguage language, FrameDescriptor fd, Signature sign, BytecodeCodeUnit co, LazySource source, boolean internal,
698+
ParserCallbacksImpl parserCallbacks) {
697699
super(language, fd);
698-
assert source != null;
699700
this.celloffset = co.varnames.length;
700701
this.freeoffset = celloffset + co.cellvars.length;
701702
this.stackoffset = freeoffset + co.freevars.length;
702703
this.bcioffset = stackoffset + co.stacksize;
703-
this.source = source;
704-
this.internal = source.isInternal();
704+
this.lazySource = source;
705+
this.internal = internal;
705706
this.parserCallbacks = parserCallbacks;
706707
this.signature = sign;
707708
this.bytecode = co.code;
@@ -788,7 +789,11 @@ public BytecodeCodeUnit getCodeUnit() {
788789
}
789790

790791
public Source getSource() {
791-
return source;
792+
return lazySource.getSource();
793+
}
794+
795+
public LazySource getLazySource() {
796+
return lazySource;
792797
}
793798

794799
public byte[] getBytecode() {
@@ -2879,7 +2884,7 @@ private Object notifyEnter(VirtualFrame virtualFrame, InstrumentationSupport ins
28792884
}
28802885

28812886
private MakeFunctionNode insertMakeFunctionNode(Node[] localNodes, int beginBci, BytecodeCodeUnit codeUnit) {
2882-
return insertChildNode(localNodes, beginBci, MakeFunctionNodeGen.class, () -> MakeFunctionNode.create(getLanguage(PythonLanguage.class), codeUnit, source));
2887+
return insertChildNode(localNodes, beginBci, MakeFunctionNodeGen.class, () -> MakeFunctionNode.create(getLanguage(PythonLanguage.class), codeUnit, lazySource, internal));
28832888
}
28842889

28852890
public void materializeContainedFunctionsForInstrumentation(Set<Class<? extends Tag>> materializedTags) {
@@ -6016,15 +6021,11 @@ public boolean frameIsVisibleToPython() {
60166021
public SourceSection getSourceSection() {
60176022
if (sourceSection != null) {
60186023
return sourceSection;
6019-
} else if (!source.hasCharacters()) {
6020-
/*
6021-
* TODO We could still expose the disassembled bytecode for a debugger to have something
6022-
* to step through.
6023-
*/
6024-
sourceSection = source.createUnavailableSection();
6024+
} else if (!getSource().hasCharacters()) {
6025+
sourceSection = getSource().createUnavailableSection();
60256026
return sourceSection;
60266027
} else {
6027-
sourceSection = co.getSourceSection(source);
6028+
sourceSection = co.getSourceSection(getSource());
60286029
return sourceSection;
60296030
}
60306031
}
@@ -6098,7 +6099,7 @@ protected boolean isCloneUninitializedSupported() {
60986099

60996100
@Override
61006101
protected RootNode cloneUninitialized() {
6101-
return new PBytecodeRootNode(getLanguage(), getFrameDescriptor(), getSignature(), co, source, parserCallbacks);
6102+
return new PBytecodeRootNode(getLanguage(), getFrameDescriptor(), getSignature(), co, lazySource, internal, parserCallbacks);
61026103
}
61036104

61046105
public void triggerDeferredDeprecationWarnings() {

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/nodes/bytecode/instrumentation/InstrumentationRootImpl.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
/*
2-
* Copyright (c) 2022, Oracle and/or its affiliates. All rights reserved.
2+
* Copyright (c) 2022, 2025, Oracle and/or its affiliates. All rights reserved.
33
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
44
*
55
* The Universal Permissive License (UPL), Version 1.0
@@ -66,7 +66,7 @@ public InstrumentableNode materializeInstrumentableNodes(Set<Class<? extends Tag
6666
node = node.getParent();
6767
}
6868
PBytecodeRootNode rootNode = (PBytecodeRootNode) node;
69-
if (rootNode.getSource() != null && rootNode.getSource().hasCharacters()) {
69+
if (rootNode.getSource().hasCharacters()) {
7070
instrumentationSupport = insert(new InstrumentationSupport(rootNode));
7171
rootNode.materializeContainedFunctionsForInstrumentation(materializedTags);
7272
notifyInserted(instrumentationSupport);

‎graalpython/com.oracle.graal.python/src/com/oracle/graal/python/util/LazySource.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
/*
2+
* Copyright (c) 2025, Oracle and/or its affiliates. All rights reserved.
3+
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4+
*
5+
* The Universal Permissive License (UPL), Version 1.0
6+
*
7+
* Subject to the condition set forth below, permission is hereby granted to any
8+
* person obtaining a copy of this software, associated documentation and/or
9+
* data (collectively the "Software"), free of charge and under any and all
10+
* copyright rights in the Software, and any and all patent rights owned or
11+
* freely licensable by each licensor hereunder covering either (i) the
12+
* unmodified Software as contributed to or provided by such licensor, or (ii)
13+
* the Larger Works (as defined below), to deal in both
14+
*
15+
* (a) the Software, and
16+
*
17+
* (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
18+
* one is included with the Software each a "Larger Work" to which the Software
19+
* is contributed by such licensors),
20+
*
21+
* without restriction, including without limitation the rights to copy, create
22+
* derivative works of, display, perform, and distribute the Software and make,
23+
* use, sell, offer for sale, import, export, have made, and have sold the
24+
* Software and the Larger Work(s), and to sublicense the foregoing rights on
25+
* either these or other terms.
26+
*
27+
* This license is subject to the following condition:
28+
*
29+
* The above copyright notice and either this complete permission notice or at a
30+
* minimum a reference to the UPL must be included in all copies or substantial
31+
* portions of the Software.
32+
*
33+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
34+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
35+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
36+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
37+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
38+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
39+
* SOFTWARE.
40+
*/
41+
package com.oracle.graal.python.util;
42+
43+
import java.io.IOException;
44+
45+
import com.oracle.truffle.api.source.Source;
46+
47+
public final class LazySource {
48+
private Source.SourceBuilder builder;
49+
private Source source;
50+
51+
public LazySource(Source.SourceBuilder builder) {
52+
this.builder = builder;
53+
}
54+
55+
public LazySource(Source source) {
56+
this.source = source;
57+
}
58+
59+
public Source getSource() {
60+
if (source == null) {
61+
try {
62+
source = builder.build();
63+
} catch (IOException e) {
64+
source = builder.content(Source.CONTENT_NONE).build();
65+
}
66+
builder = null;
67+
}
68+
return source;
69+
}
70+
}

0 commit comments

Comments
 (0)