experiment: Deadlock logger

CHANGELOG.md

@@ -10,6 +10,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [11.3.0]
 
 - Adds SAML features
+- Fixes potential deadlock issue with `TelemetryProvider`
+- Adds DeadlockLogger as an utility for discovering deadlock issues
 
 ### Migration
 
@@ -68,6 +70,11 @@ CREATE INDEX IF NOT EXISTS saml_claims_app_id_tenant_id_index ON saml_claims (ap
 CREATE INDEX IF NOT EXISTS saml_claims_expires_at_index ON saml_claims (expires_at);
 ```
 
+## [11.2.1]
+
+- Fixes deadlock issue with `ResourceDistributor`
+- Fixes race issues with Refreshing OAuth token
+
 ## [11.2.0]
 
 - Adds opentelemetry-javaagent to the core distribution

config.yaml

@@ -187,6 +187,9 @@ core_config_version: 0
 # will send telemetry data. This should be in the format http://<host>:<port> or https://<host>:<port>.
 # otel_collector_connection_uri:
 
+# (OPTIONAL | Default: false) boolean value. Enables or disables the deadlock logger.
+# deadlock_logger_enable:
+
 # (OPTIONAL | Default: null) string value. If specified, uses this URL as ACS URL for handling legacy SAML clients
 # saml_legacy_acs_url:
 

devConfig.yaml

@@ -187,6 +187,9 @@ disable_telemetry: true
 # will send telemetry data. This should be in the format http://<host>:<port> or https://<host>:<port>.
 # otel_collector_connection_uri:
 
+# (OPTIONAL | Default: false) boolean value. Enables or disables the deadlock logger.
+# deadlock_logger_enable:
+
 # (OPTIONAL | Default: null) string value. If specified, uses this URL as ACS URL for handling legacy SAML clients
 saml_legacy_acs_url: "http://localhost:5225/api/oauth/saml"
 

src/main/java/io/supertokens/Main.java

@@ -24,6 +24,7 @@
 import io.supertokens.cronjobs.cleanupOAuthSessionsAndChallenges.CleanupOAuthSessionsAndChallenges;
 import io.supertokens.cronjobs.deleteExpiredSAMLData.DeleteExpiredSAMLData;
 import io.supertokens.cronjobs.cleanupWebauthnExpiredData.CleanUpWebauthNExpiredDataCron;
+import io.supertokens.cronjobs.deadlocklogger.DeadlockLogger;
 import io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys;
 import io.supertokens.cronjobs.deleteExpiredDashboardSessions.DeleteExpiredDashboardSessions;
 import io.supertokens.cronjobs.deleteExpiredEmailVerificationTokens.DeleteExpiredEmailVerificationTokens;
@@ -283,6 +284,11 @@ private void init() throws IOException, StorageQueryException {
 
         Cronjobs.addCronjob(this, CleanUpWebauthNExpiredDataCron.init(this, uniqueUserPoolIdsTenants));
 
+        // starts the DeadlockLogger if
+        if (Config.getBaseConfig(this).isDeadlockLoggerEnabled()) {
+            DeadlockLogger.getInstance().start();
+        }
+
         Cronjobs.addCronjob(this, DeleteExpiredSAMLData.init(this, uniqueUserPoolIdsTenants));
 
         // this is to ensure tenantInfos are in sync for the new cron job as well

src/main/java/io/supertokens/ResourceDistributor.java

@@ -51,12 +51,12 @@ public static TenantIdentifier getAppForTesting() {
         return appUsedForTesting;
     }
 
-    public synchronized SingletonResource getResource(AppIdentifier appIdentifier, @Nonnull String key)
+    public SingletonResource getResource(AppIdentifier appIdentifier, @Nonnull String key)
             throws TenantOrAppNotFoundException {
         return getResource(appIdentifier.getAsPublicTenantIdentifier(), key);
     }
 
-    public synchronized SingletonResource getResource(TenantIdentifier tenantIdentifier, @Nonnull String key)
+    public SingletonResource getResource(TenantIdentifier tenantIdentifier, @Nonnull String key)
             throws TenantOrAppNotFoundException {
         // first we do exact match
         SingletonResource resource = resources.get(new KeyClass(tenantIdentifier, key));
@@ -70,14 +70,6 @@ public synchronized SingletonResource getResource(TenantIdentifier tenantIdentif
             throw new TenantOrAppNotFoundException(tenantIdentifier);
         }
 
-        MultitenancyHelper.getInstance(main).refreshTenantsInCoreBasedOnChangesInCoreConfigOrIfTenantListChanged(true);
-
-        // we try again..
-        resource = resources.get(new KeyClass(tenantIdentifier, key));
-        if (resource != null) {
-            return resource;
-        }
-
         // then we see if the user has configured anything to do with connectionUriDomain, and if they have,
         // then we must return null cause the user has not specifically added tenantId to it
         for (KeyClass currKey : resources.keySet()) {
@@ -101,11 +93,11 @@ public synchronized SingletonResource getResource(TenantIdentifier tenantIdentif
     }
 
     @TestOnly
-    public synchronized SingletonResource getResource(@Nonnull String key) {
+    public SingletonResource getResource(@Nonnull String key) {
         return resources.get(new KeyClass(appUsedForTesting, key));
     }
 
-    public synchronized SingletonResource setResource(TenantIdentifier tenantIdentifier,
+    public SingletonResource setResource(TenantIdentifier tenantIdentifier,
                                                       @Nonnull String key,
                                                       SingletonResource resource) {
         SingletonResource alreadyExists = resources.get(new KeyClass(tenantIdentifier, key));
@@ -116,7 +108,7 @@ public synchronized SingletonResource setResource(TenantIdentifier tenantIdentif
         return resource;
     }
 
-    public synchronized SingletonResource removeResource(TenantIdentifier tenantIdentifier,
+    public SingletonResource removeResource(TenantIdentifier tenantIdentifier,
                                                          @Nonnull String key) {
         SingletonResource singletonResource = resources.get(new KeyClass(tenantIdentifier, key));
         if (singletonResource == null) {
@@ -126,18 +118,18 @@ public synchronized SingletonResource removeResource(TenantIdentifier tenantIden
         return singletonResource;
     }
 
-    public synchronized SingletonResource setResource(AppIdentifier appIdentifier,
+    public SingletonResource setResource(AppIdentifier appIdentifier,
                                                       @Nonnull String key,
                                                       SingletonResource resource) {
         return setResource(appIdentifier.getAsPublicTenantIdentifier(), key, resource);
     }
 
-    public synchronized SingletonResource removeResource(AppIdentifier appIdentifier,
+    public SingletonResource removeResource(AppIdentifier appIdentifier,
                                                          @Nonnull String key) {
         return removeResource(appIdentifier.getAsPublicTenantIdentifier(), key);
     }
 
-    public synchronized void clearAllResourcesWithResourceKey(String inputKey) {
+    public void clearAllResourcesWithResourceKey(String inputKey) {
         List<KeyClass> toRemove = new ArrayList<>();
         resources.forEach((key, value) -> {
             if (key.key.equals(inputKey)) {
@@ -149,7 +141,7 @@ public synchronized void clearAllResourcesWithResourceKey(String inputKey) {
         }
     }
 
-    public synchronized Map<KeyClass, SingletonResource> getAllResourcesWithResourceKey(String inputKey) {
+    public Map<KeyClass, SingletonResource> getAllResourcesWithResourceKey(String inputKey) {
         Map<KeyClass, SingletonResource> result = new HashMap<>();
         resources.forEach((key, value) -> {
             if (key.key.equals(inputKey)) {
@@ -160,7 +152,7 @@ public synchronized Map<KeyClass, SingletonResource> getAllResourcesWithResource
     }
 
     @TestOnly
-    public synchronized SingletonResource setResource(@Nonnull String key,
+    public SingletonResource setResource(@Nonnull String key,
                                                       SingletonResource resource) {
         return setResource(appUsedForTesting, key, resource);
     }

src/main/java/io/supertokens/config/CoreConfig.java

@@ -426,6 +426,13 @@ public class CoreConfig {
                     "null)")
     private String otel_collector_connection_uri = null;
 
+    @EnvName("DEADLOCK_LOGGER_ENABLE")
+    @ConfigYamlOnly
+    @JsonProperty
+    @ConfigDescription(
+            "Enables or disables the deadlock logger. (Default: false)")
+    private boolean deadlock_logger_enable = false;
+
     @IgnoreForAnnotationCheck
     private static boolean disableOAuthValidationForTest = false;
 
@@ -681,6 +688,10 @@ public String getOtelCollectorConnectionURI() {
         return otel_collector_connection_uri;
     }
 
+    public boolean isDeadlockLoggerEnabled() {
+        return deadlock_logger_enable;
+    }
+
     public String getSAMLLegacyACSURL() {
         return saml_legacy_acs_url;
     }

src/main/java/io/supertokens/cronjobs/deadlocklogger/DeadlockLogger.java

@@ -0,0 +1,84 @@
+/*
+ *    Copyright (c) 2025, VRAI Labs and/or its affiliates. All rights reserved.
+ *
+ *    This software is licensed under the Apache License, Version 2.0 (the
+ *    "License") as published by the Apache Software Foundation.
+ *
+ *    You may not use this file except in compliance with the License. You may
+ *    obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ *    Unless required by applicable law or agreed to in writing, software
+ *    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ *    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ *    License for the specific language governing permissions and limitations
+ *    under the License.
+ */
+
+package io.supertokens.cronjobs.deadlocklogger;
+
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadInfo;
+import java.lang.management.ThreadMXBean;
+import java.util.Arrays;
+
+public class DeadlockLogger {
+
+    private static final DeadlockLogger INSTANCE = new DeadlockLogger();
+
+    private DeadlockLogger() {
+    }
+
+    public static DeadlockLogger getInstance() {
+        return INSTANCE;
+    }
+
+    public void start(){
+        Thread deadlockLoggerThread = new Thread(deadlockDetector, "DeadlockLoggerThread");
+        deadlockLoggerThread.setDaemon(true);
+        deadlockLoggerThread.start();
+    }
+
+    private final Runnable deadlockDetector = new Runnable() {
+        @Override
+        public void run() {
+            System.out.println("DeadlockLogger started!");
+            while (true) {
+                System.out.println("DeadlockLogger - checking");
+                ThreadMXBean bean = ManagementFactory.getThreadMXBean();
+                long[] threadIds = bean.findDeadlockedThreads(); // Returns null if no threads are deadlocked.
+                System.out.println("DeadlockLogger - DeadlockedThreads: " + Arrays.toString(threadIds));
+                if (threadIds != null) {
+                    ThreadInfo[] infos = bean.getThreadInfo(threadIds);
+                    boolean deadlockFound = false;
+                    System.out.println("DEADLOCK found!");
+                    for (ThreadInfo info : infos) {
+                        System.out.println("ThreadName: " + info.getThreadName());
+                        System.out.println("Thread ID: " + info.getThreadId());
+                        System.out.println("LockName: " + info.getLockName());
+                        System.out.println("LockOwnerName: " + info.getLockOwnerName());
+                        System.out.println("LockedMonitors: " + Arrays.toString(info.getLockedMonitors()));
+                        System.out.println("LockInfo: " + info.getLockInfo());
+                        System.out.println("Stack: " + Arrays.toString(info.getStackTrace()));
+                        System.out.println();
+                        deadlockFound = true;
+                    }
+                    System.out.println("*******************************");
+                    if(deadlockFound) {
+                        System.out.println(" ==== ALL THREAD INFO ===");
+                        ThreadInfo[] allThreads = bean.dumpAllThreads(true, true, 100);
+                        for (ThreadInfo threadInfo : allThreads) {
+                            System.out.println("THREAD: " + threadInfo.getThreadName());
+                            System.out.println("StackTrace: " + Arrays.toString(threadInfo.getStackTrace()));
+                        }
+                        break;
+                    }
+                }
+                try {
+                    Thread.sleep(10000);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        }
+    };
+}

src/main/java/io/supertokens/telemetry/TelemetryProvider.java

@@ -48,7 +48,7 @@ public class TelemetryProvider extends ResourceDistributor.SingletonResource imp
 
     private final OpenTelemetry openTelemetry;
 
-    public static synchronized TelemetryProvider getInstance(Main main) {
+    public static TelemetryProvider getInstance(Main main) {
         TelemetryProvider instance = null;
         try {
             instance = (TelemetryProvider) main.getResourceDistributor()