[GR-70846] Ensure core types can be treated as closed

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/PointsToAnalysis.java

@@ -175,8 +175,11 @@ public boolean isClosed(AnalysisType type) {
             /* In a closed type world all subtypes known. */
             return true;
         }
-        /* Array and leaf types are by definition closed. */
-        return type.isArray() || type.isLeaf();
+        /*
+         * Array and leaf types are by definition closed. Core types are considered as closed to
+         * allow optimizations to be applied on them.
+         */
+        return type.isArray() || type.isLeaf() || hostVM.isCoreType(type);
     }
 
     @Override

substratevm/src/com.oracle.graal.pointsto/src/com/oracle/graal/pointsto/results/TypeFlowSimplifier.java

@@ -347,18 +347,13 @@ private void handleInvoke(Invoke invoke, SimplifierTool tool) {
          * 2. The receiver TypeFlow's type is a closed type, so it may be not extended in a later
          * layer.
          *
-         * 3. The receiver TypeFlow's type is a core type, so it may be not extended in a later
-         * layer. (GR-70846: This check condition will be merged with the previous one once core
-         * types are fully considered as closed.)
-         *
-         * 4. The receiver TypeFlow is not saturated.
+         * 3. The receiver TypeFlow is not saturated.
          *
          * Otherwise, when the receiver's analysis results are incomplete, then it is possible for
          * more types to be observed in subsequent layers.
          */
         boolean receiverAnalysisResultsComplete = strengthenGraphs.isClosedTypeWorld ||
-                        (hasReceiver && (analysis.isClosed(invokeFlow.getReceiverType()) || analysis.getHostVM().isCoreType(invokeFlow.getReceiverType()) ||
-                                        !methodFlow.isSaturated(analysis, invokeFlow.getReceiver())));
+                        (hasReceiver && (analysis.isClosed(invokeFlow.getReceiverType()) || !methodFlow.isSaturated(analysis, invokeFlow.getReceiver())));
 
         if (callTarget.invokeKind().isDirect()) {
             /*

substratevm/src/com.oracle.svm.common/src/com/oracle/svm/common/layeredimage/LayeredCompilationBehavior.java

@@ -33,7 +33,7 @@
  * Used to specify how a method needs to be compiled when building layered images.
  */
 @Retention(RetentionPolicy.RUNTIME)
-@Target({ElementType.METHOD})
+@Target({ElementType.METHOD, ElementType.CONSTRUCTOR})
 public @interface LayeredCompilationBehavior {
     /**
      * This state represents how a method should be compiled in layered images. The state of a

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/InvalidMethodPointerHandler.java

@@ -35,6 +35,8 @@
 import org.graalvm.nativeimage.c.function.CodePointer;
 import org.graalvm.word.Pointer;
 
+import com.oracle.svm.common.layeredimage.LayeredCompilationBehavior;
+import com.oracle.svm.common.layeredimage.LayeredCompilationBehavior.Behavior;
 import com.oracle.svm.core.graal.code.StubCallingConvention;
 import com.oracle.svm.core.heap.RestrictHeapAccess;
 import com.oracle.svm.core.log.Log;
@@ -79,6 +81,7 @@ private static void invalidCodeAddressHandler() {
     @StubCallingConvention
     @NeverInline("We need a separate frame that stores all registers")
     @Uninterruptible(reason = "Precaution.")
+    @LayeredCompilationBehavior(Behavior.FULLY_DELAYED_TO_APPLICATION_LAYER)
     private static void invalidVTableEntryHandler() {
         Pointer callerSP = KnownIntrinsics.readCallerStackPointer();
         CodePointer callerIP = KnownIntrinsics.readReturnAddress();

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/JavaMainWrapper.java

@@ -56,6 +56,7 @@
 import org.graalvm.word.WordBase;
 
 import com.oracle.svm.common.layeredimage.LayeredCompilationBehavior;
+import com.oracle.svm.common.layeredimage.LayeredCompilationBehavior.Behavior;
 import com.oracle.svm.core.c.CGlobalData;
 import com.oracle.svm.core.c.CGlobalDataFactory;
 import com.oracle.svm.core.c.function.CEntryPointActions;
@@ -166,7 +167,7 @@ public List<String> getInputArguments() {
      * For layered images this method is delayed until the application layer. This is necessary so
      * that the method handle can be inlined before analysis.
      */
-    @LayeredCompilationBehavior(LayeredCompilationBehavior.Behavior.FULLY_DELAYED_TO_APPLICATION_LAYER)
+    @LayeredCompilationBehavior(Behavior.FULLY_DELAYED_TO_APPLICATION_LAYER)
     public static void invokeMain(String[] args) throws Throwable {
         String[] mainArgs = args;
         if (ImageSingletons.contains(PreMainSupport.class)) {
@@ -276,9 +277,11 @@ private static void runShutdown0() {
         }
     }
 
+    /** The entry point of the image needs to be in the application layer. */
     @Uninterruptible(reason = "Thread state not set up yet.")
     @CEntryPoint(include = CEntryPoint.NotIncludedAutomatically.class)
     @CEntryPointOptions(prologue = NoPrologue.class, epilogue = NoEpilogue.class)
+    @LayeredCompilationBehavior(Behavior.FULLY_DELAYED_TO_APPLICATION_LAYER)
     public static int run(int argc, CCharPointerPointer argv) {
         if (SubstrateOptions.RunMainInNewThread.getValue()) {
             return doRunInNewThread(argc, argv);
@@ -289,7 +292,7 @@ public static int run(int argc, CCharPointerPointer argv) {
 
     /** SVM start-up logic should be pinned to the initial layer. */
     @Uninterruptible(reason = "Thread state not setup yet.")
-    @LayeredCompilationBehavior(LayeredCompilationBehavior.Behavior.PINNED_TO_INITIAL_LAYER)
+    @LayeredCompilationBehavior(Behavior.PINNED_TO_INITIAL_LAYER)
     private static int doRun(int argc, CCharPointerPointer argv) {
         try {
             CPUFeatureAccess cpuFeatureAccess = ImageSingletons.lookup(CPUFeatureAccess.class);

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/management/ManagementSupport.java

@@ -53,6 +53,8 @@
 import org.graalvm.nativeimage.Platforms;
 import org.graalvm.nativeimage.impl.InternalPlatform;
 
+import com.oracle.svm.common.layeredimage.LayeredCompilationBehavior;
+import com.oracle.svm.common.layeredimage.LayeredCompilationBehavior.Behavior;
 import com.oracle.svm.core.GCRelatedMXBeans;
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.Uninterruptible;
@@ -136,6 +138,7 @@ public static ManagementSupport getSingleton() {
         return ImageSingletons.lookup(ManagementSupport.class);
     }
 
+    @LayeredCompilationBehavior(Behavior.PINNED_TO_INITIAL_LAYER)
     public <T extends PlatformManagedObject> T getPlatformMXBean(Class<T> clazz) {
         Object result = getPlatformMXBeans0(clazz);
         if (result == null) {
@@ -152,6 +155,7 @@ public PlatformManagedObject getPlatformMXBeanRaw(Class<? extends PlatformManage
     }
 
     @SuppressWarnings("unchecked")
+    @LayeredCompilationBehavior(Behavior.PINNED_TO_INITIAL_LAYER)
     public <T extends PlatformManagedObject> List<T> getPlatformMXBeans(Class<T> clazz) {
         Object result = getPlatformMXBeans0(clazz);
         if (result == null) {

substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/reflect/serialize/SerializationSupport.java

@@ -40,8 +40,10 @@
 import org.graalvm.nativeimage.dynamicaccess.AccessCondition;
 
 import com.oracle.svm.core.BuildPhaseProvider;
+import com.oracle.svm.core.BuildPhaseProvider.AfterCompilation;
 import com.oracle.svm.core.SubstrateUtil;
 import com.oracle.svm.core.configure.RuntimeDynamicAccessMetadata;
+import com.oracle.svm.core.heap.UnknownObjectField;
 import com.oracle.svm.core.hub.DynamicHub;
 import com.oracle.svm.core.layeredimagesingleton.LayeredImageSingletonSupport;
 import com.oracle.svm.core.layeredimagesingleton.MultiLayeredImageSingleton;
@@ -209,12 +211,14 @@ public String getClassLoaderSerializationLookupKey(ClassLoader classLoader) {
      * Temporary key for maps ideally indexed by their {@link Class} or {@link DynamicHub}. At
      * runtime, these maps should be indexed by {@link DynamicHub#getTypeID}
      */
+    @Platforms(Platform.HOSTED_ONLY.class)
     public record DynamicHubKey(DynamicHub hub) {
         public int getTypeID() {
             return hub.getTypeID();
         }
     }
 
+    @UnknownObjectField(fullyQualifiedTypes = "org.graalvm.collections.EconomicMapImpl", availability = AfterCompilation.class) //
     private final EconomicMap<Object /* DynamicHubKey or DynamicHub.typeID */, RuntimeDynamicAccessMetadata> classes = EconomicMap.create();
     private final EconomicMap<String, RuntimeDynamicAccessMetadata> lambdaCapturingClasses = EconomicMap.create();
 

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/ExtensionLayerImageFeature.java

@@ -24,20 +24,41 @@
  */
 package com.oracle.svm.hosted;
 
+import java.util.Arrays;
+import java.util.Objects;
+import java.util.function.Predicate;
+
+import org.graalvm.collections.EconomicMap;
+
+import com.oracle.graal.pointsto.api.HostVM;
+import com.oracle.graal.pointsto.meta.AnalysisType;
+import com.oracle.graal.pointsto.meta.AnalysisUniverse;
+import com.oracle.svm.common.layeredimage.LayeredCompilationBehavior;
 import com.oracle.svm.core.c.BoxedRelocatedPointer;
 import com.oracle.svm.core.code.ImageCodeInfo;
 import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.feature.InternalFeature;
 import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
+import com.oracle.svm.core.traits.BuiltinTraits.BuildtimeAccessOnly;
+import com.oracle.svm.core.traits.BuiltinTraits.NoLayeredCallbacks;
+import com.oracle.svm.core.traits.SingletonLayeredInstallationKind.Independent;
+import com.oracle.svm.core.traits.SingletonTraits;
+import com.oracle.svm.core.util.VMError;
 import com.oracle.svm.hosted.FeatureImpl.BeforeAnalysisAccessImpl;
+import com.oracle.svm.hosted.FeatureImpl.BeforeCompilationAccessImpl;
+import com.oracle.svm.hosted.imagelayer.HostedImageLayerBuildingSupport;
+import com.oracle.svm.hosted.imagelayer.InitialLayerFeature;
+import com.oracle.svm.hosted.imagelayer.SVMImageLayerLoader;
 import com.oracle.svm.util.ReflectionUtil;
 
 /**
  * This feature contains some configs currently necessary to build an extension layer. We'll need
  * better mechanisms to avoid these workarounds.
  */
 @AutomaticallyRegisteredFeature
+@SingletonTraits(access = BuildtimeAccessOnly.class, layeredCallbacks = NoLayeredCallbacks.class, layeredInstallationKind = Independent.class)
 final class ExtensionLayerImageFeature implements InternalFeature {
+    private static final Object NULL_SUPER_CORE_TYPE = new Object();
 
     @Override
     public boolean isInConfiguration(IsInConfigurationAccess access) {
@@ -72,4 +93,83 @@ public void beforeAnalysis(BeforeAnalysisAccess a) {
          */
         access.registerAsInHeap(ReflectionUtil.lookupClass(false, "java.util.concurrent.ConcurrentHashMap$CounterCell"));
     }
+
+    @Override
+    public void beforeCompilation(BeforeCompilationAccess a) {
+        BeforeCompilationAccessImpl access = (BeforeCompilationAccessImpl) a;
+        AnalysisUniverse universe = access.aUniverse;
+        HostVM hostVM = universe.hostVM();
+        SVMImageLayerLoader imageLayerLoader = HostedImageLayerBuildingSupport.singleton().getLoader();
+        EconomicMap<AnalysisType, Object> superCoreTypes = EconomicMap.create();
+
+        for (var type : universe.getTypes()) {
+            /*
+             * These checks allow to ensure that core types can be treated as closed (see
+             * PointsToAnalysis.isClosed(AnalysisType)).
+             */
+            boolean coreType = hostVM.isCoreType(type);
+            if (coreType) {
+                superCoreTypes.put(type, type);
+            }
+            AnalysisType superCoreType = getSuperCoreType(type, hostVM, superCoreTypes);
+            boolean extendsCoreType = superCoreType != null && !coreType;
+            if (coreType || extendsCoreType) {
+                /*
+                 * This checks that all core types and types that extend or implement a core type
+                 * were not marked as reachable or instantiated only in an extension layer, showing
+                 * that the analysis of the core was complete.
+                 */
+                checkCondition(type.isReachable(), imageLayerLoader::isReachableInPreviousLayer, type, extendsCoreType, superCoreType, "reachable");
+                checkCondition(type.isInstantiated(), imageLayerLoader::isInstantiatedInPreviousLayer, type, extendsCoreType, superCoreType, "instantiated");
+            }
+        }
+    }
+
+    public AnalysisType getSuperCoreType(AnalysisType type, HostVM hostVM, EconomicMap<AnalysisType, Object> superCoreTypes) {
+        /* Check the cache to see if the super core type was already computed. */
+        if (superCoreTypes.containsKey(type)) {
+            Object result = superCoreTypes.get(type);
+            return result == NULL_SUPER_CORE_TYPE ? null : (AnalysisType) result;
+        }
+
+        AnalysisType result = null;
+
+        /* Go through the super types to check if one of them is a core type. */
+        AnalysisType superType = type.getSuperclass();
+        if (superType != null) {
+            result = hostVM.isCoreType(superType) ? superType : getSuperCoreType(superType, hostVM, superCoreTypes);
+        }
+
+        if (result == null) {
+            /*
+             * If no result was found, iterate through all interfaces to see if one of them is a
+             * core type.
+             */
+            AnalysisType[] interfaces = type.getInterfaces();
+            var coreInterface = Arrays.stream(interfaces).map(i -> getSuperCoreType(i, hostVM, superCoreTypes)).filter(Objects::nonNull).findFirst();
+            if (coreInterface.isPresent()) {
+                result = coreInterface.get();
+            }
+        }
+
+        /* Cache the result. */
+        superCoreTypes.put(type, result == null ? NULL_SUPER_CORE_TYPE : result);
+
+        return result;
+    }
+
+    private static void checkCondition(boolean condition, Predicate<AnalysisType> test, AnalysisType type, boolean extendsCoreType, AnalysisType superCoreType, String kind) {
+        if (condition) {
+            String hint = "Please make sure that all core types are seen by the initial layer analysis by either registering more entry points using " +
+                            LayeredCompilationBehavior.class + " with the " + LayeredCompilationBehavior.Behavior.PINNED_TO_INITIAL_LAYER +
+                            " value or by explicitly registering the entry point or type reachability in " + InitialLayerFeature.class;
+            if (extendsCoreType) {
+                VMError.guarantee(test.test(type), "The type %s is extending the core type %s which was not seen as %s the initial layer. " +
+                                "It is illegal to extend core types in subsequent layers. %s", type, superCoreType, kind, hint);
+            } else {
+                VMError.guarantee(test.test(type), "The core type %s was not seen as %s the initial layer. " +
+                                "It is illegal for core types to become reachable in subsequent layers. %s", type, kind, hint);
+            }
+        }
+    }
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/InitialLayerFeature.java

@@ -26,14 +26,21 @@
 
 import static com.oracle.svm.common.layeredimage.LayeredCompilationBehavior.Behavior.PINNED_TO_INITIAL_LAYER;
 
+import java.lang.reflect.Proxy;
+
 import org.graalvm.nativeimage.hosted.Feature;
 
+import com.oracle.graal.pointsto.meta.AnalysisMetaAccess;
 import com.oracle.svm.common.hosted.layeredimage.LayeredCompilationSupport;
+import com.oracle.svm.core.Uninterruptible;
+import com.oracle.svm.core.bootstrap.BootstrapMethodInfo;
 import com.oracle.svm.core.feature.AutomaticallyRegisteredFeature;
 import com.oracle.svm.core.feature.InternalFeature;
 import com.oracle.svm.core.imagelayer.ImageLayerBuildingSupport;
+import com.oracle.svm.hosted.FeatureImpl.DuringSetupAccessImpl;
 import com.oracle.svm.util.ReflectionUtil;
 
+import jdk.internal.loader.ClassLoaders;
 import jdk.internal.misc.Unsafe;
 
 @AutomaticallyRegisteredFeature
@@ -43,8 +50,10 @@ public boolean isInConfiguration(Feature.IsInConfigurationAccess access) {
         return ImageLayerBuildingSupport.buildingInitialLayer();
     }
 
+    @SuppressWarnings("deprecation")
     @Override
-    public void duringSetup(DuringSetupAccess access) {
+    public void duringSetup(DuringSetupAccess a) {
+        DuringSetupAccessImpl access = (DuringSetupAccessImpl) a;
         /*
          * Make sure that critical VM components are included in the base layer by registering
          * runtime APIs as entry points. Although the types below are part of java.base, so they
@@ -57,6 +66,12 @@ public void duringSetup(DuringSetupAccess access) {
         compilationSupport.registerCompilationBehavior(ReflectionUtil.lookupMethod(Runtime.class, "getRuntime"), PINNED_TO_INITIAL_LAYER);
         compilationSupport.registerCompilationBehavior(ReflectionUtil.lookupMethod(Runtime.class, "gc"), PINNED_TO_INITIAL_LAYER);
         compilationSupport.registerCompilationBehavior(ReflectionUtil.lookupMethod(Class.class, "getResource", String.class), PINNED_TO_INITIAL_LAYER);
+
+        AnalysisMetaAccess metaAccess = access.getMetaAccess();
+        metaAccess.lookupJavaType(Uninterruptible.class).registerAsReachable("Core type");
+        metaAccess.lookupJavaType(Proxy.getProxyClass(ClassLoaders.appClassLoader(), Uninterruptible.class)).registerAsInstantiated("Core type");
+        metaAccess.lookupJavaType(BootstrapMethodInfo.class).registerAsInstantiated("Core type");
+        metaAccess.lookupJavaType(BootstrapMethodInfo.ExceptionWrapper.class).registerAsInstantiated("Core type");
     }
 
 }

substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/SVMImageLayerLoader.java

@@ -109,6 +109,7 @@
 import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.PersistedAnalysisMethod.WrappedMethod;
 import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.PersistedAnalysisMethod.WrappedMethod.WrappedMember;
 import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.PersistedAnalysisType;
+import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.PersistedAnalysisType.Reader;
 import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.PersistedAnalysisType.WrappedType;
 import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.PersistedAnalysisType.WrappedType.SerializationGenerated;
 import com.oracle.svm.hosted.imagelayer.SharedLayerSnapshotCapnProtoSchemaHolder.PersistedAnnotationElement;
@@ -1858,6 +1859,36 @@ private void scanCompanionField(DynamicHub hub) {
         instance.readFieldValue(metaAccess.lookupJavaField(dynamicHubCompanionField));
     }
 
+    public boolean isReachableInPreviousLayer(AnalysisType type) {
+        return getPropertyInPreviousLayer(type, Reader::getIsReachable);
+    }
+
+    public boolean isInstantiatedInPreviousLayer(AnalysisType type) {
+        return getPropertyInPreviousLayer(type, Reader::getIsInstantiated);
+    }
+
+    private boolean getPropertyInPreviousLayer(AnalysisType type, Function<PersistedAnalysisType.Reader, Boolean> propertyGetter) {
+        Integer typeId;
+        if (type.getWrapped() instanceof BaseLayerType baseLayerType) {
+            typeId = baseLayerType.getBaseLayerId();
+        } else {
+            /*
+             * Types that cannot be loaded manually can be duplicated and can get a new type id even
+             * if they were in a shared layer. In this case, using the type identifier can still
+             * retrieve the id from the shared layer.
+             */
+            String typeDescriptor = imageLayerSnapshotUtil.getTypeDescriptor(type);
+            typeId = typeDescriptorToBaseLayerId.get(typeDescriptor);
+        }
+        if (typeId != null) {
+            var typeInfo = findType(typeId);
+            if (typeInfo != null) {
+                return propertyGetter.apply(typeInfo);
+            }
+        }
+        return false;
+    }
+
     public record LayeredSimulationResult(boolean successful, EconomicMap<AnalysisField, JavaConstant> staticFieldValues) {
     }