experiment: Deadlock logger (#1198)

* experiment: Deadlock logger

* fix: race issue with oauth refresh (#1199)

* fix: race issue with oauth refresh

* fix: review comment

* fix: remove print

* fix: deadlock in resource distributor (#1197)

* adding dev-v11.2.1 tag to this commit to ensure building

* fix: add deadlock logger

* fix: changelog and build version

* fix: only start deadlocklogger if it's enabled

---------

Co-authored-by: Sattvik Chakravarthy <sattvik@supertokens.com>
Co-authored-by: Supertokens Bot <>

Author: tamassoltesz

CHANGELOG.md

@@ -10,6 +10,8 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [11.3.0]
 
 - Adds SAML features
+- Fixes potential deadlock issue with `TelemetryProvider`
+- Adds DeadlockLogger as an utility for discovering deadlock issues
 
 ### Migration
 
@@ -68,6 +70,11 @@ CREATE INDEX IF NOT EXISTS saml_claims_app_id_tenant_id_index ON saml_claims (ap
 CREATE INDEX IF NOT EXISTS saml_claims_expires_at_index ON saml_claims (expires_at);
 ```
 
+## [11.2.1]
+
+- Fixes deadlock issue with `ResourceDistributor`
+- Fixes race issues with Refreshing OAuth token
+
 ## [11.2.0]
 
 - Adds opentelemetry-javaagent to the core distribution

cli/jar/cli.jar

@@ -187,6 +187,9 @@ core_config_version: 0
 # will send telemetry data. This should be in the format http://<host>:<port> or https://<host>:<port>.
 # otel_collector_connection_uri:
 
+# (OPTIONAL | Default: false) boolean value. Enables or disables the deadlock logger.
+# deadlock_logger_enable:
+
 # (OPTIONAL | Default: null) string value. If specified, uses this URL as ACS URL for handling legacy SAML clients
 # saml_legacy_acs_url:
 

config.yaml

@@ -187,6 +187,9 @@ disable_telemetry: true
 # will send telemetry data. This should be in the format http://<host>:<port> or https://<host>:<port>.
 # otel_collector_connection_uri:
 
+# (OPTIONAL | Default: false) boolean value. Enables or disables the deadlock logger.
+# deadlock_logger_enable:
+
 # (OPTIONAL | Default: null) string value. If specified, uses this URL as ACS URL for handling legacy SAML clients
 saml_legacy_acs_url: "http://localhost:5225/api/oauth/saml"
 

devConfig.yaml

@@ -24,6 +24,7 @@
 import io.supertokens.cronjobs.cleanupOAuthSessionsAndChallenges.CleanupOAuthSessionsAndChallenges;
 import io.supertokens.cronjobs.deleteExpiredSAMLData.DeleteExpiredSAMLData;
 import io.supertokens.cronjobs.cleanupWebauthnExpiredData.CleanUpWebauthNExpiredDataCron;
+import io.supertokens.cronjobs.deadlocklogger.DeadlockLogger;
 import io.supertokens.cronjobs.deleteExpiredAccessTokenSigningKeys.DeleteExpiredAccessTokenSigningKeys;
 import io.supertokens.cronjobs.deleteExpiredDashboardSessions.DeleteExpiredDashboardSessions;
 import io.supertokens.cronjobs.deleteExpiredEmailVerificationTokens.DeleteExpiredEmailVerificationTokens;
@@ -283,6 +284,11 @@ private void init() throws IOException, StorageQueryException {
 
         Cronjobs.addCronjob(this, CleanUpWebauthNExpiredDataCron.init(this, uniqueUserPoolIdsTenants));
 
+        // starts the DeadlockLogger if
+        if (Config.getBaseConfig(this).isDeadlockLoggerEnabled()) {
+            DeadlockLogger.getInstance().start();
+        }
+      
         Cronjobs.addCronjob(this, DeleteExpiredSAMLData.init(this, uniqueUserPoolIdsTenants));
 
         // this is to ensure tenantInfos are in sync for the new cron job as well

downloader/jar/downloader.jar

@@ -51,12 +51,12 @@ public static TenantIdentifier getAppForTesting() {
         return appUsedForTesting;
     }
 
-    public synchronized SingletonResource getResource(AppIdentifier appIdentifier, @Nonnull String key)
+    public SingletonResource getResource(AppIdentifier appIdentifier, @Nonnull String key)
             throws TenantOrAppNotFoundException {
         return getResource(appIdentifier.getAsPublicTenantIdentifier(), key);
     }
 
-    public synchronized SingletonResource getResource(TenantIdentifier tenantIdentifier, @Nonnull String key)
+    public SingletonResource getResource(TenantIdentifier tenantIdentifier, @Nonnull String key)
             throws TenantOrAppNotFoundException {
         // first we do exact match
         SingletonResource resource = resources.get(new KeyClass(tenantIdentifier, key));
@@ -70,14 +70,6 @@ public synchronized SingletonResource getResource(TenantIdentifier tenantIdentif
             throw new TenantOrAppNotFoundException(tenantIdentifier);
         }
 
-        MultitenancyHelper.getInstance(main).refreshTenantsInCoreBasedOnChangesInCoreConfigOrIfTenantListChanged(true);
-
-        // we try again..
-        resource = resources.get(new KeyClass(tenantIdentifier, key));
-        if (resource != null) {
-            return resource;
-        }
-
         // then we see if the user has configured anything to do with connectionUriDomain, and if they have,
         // then we must return null cause the user has not specifically added tenantId to it
         for (KeyClass currKey : resources.keySet()) {
@@ -101,11 +93,11 @@ public synchronized SingletonResource getResource(TenantIdentifier tenantIdentif
     }
 
     @TestOnly
-    public synchronized SingletonResource getResource(@Nonnull String key) {
+    public SingletonResource getResource(@Nonnull String key) {
         return resources.get(new KeyClass(appUsedForTesting, key));
     }
 
-    public synchronized SingletonResource setResource(TenantIdentifier tenantIdentifier,
+    public SingletonResource setResource(TenantIdentifier tenantIdentifier,
                                                       @Nonnull String key,
                                                       SingletonResource resource) {
         SingletonResource alreadyExists = resources.get(new KeyClass(tenantIdentifier, key));
@@ -116,7 +108,7 @@ public synchronized SingletonResource setResource(TenantIdentifier tenantIdentif
         return resource;
     }
 
-    public synchronized SingletonResource removeResource(TenantIdentifier tenantIdentifier,
+    public SingletonResource removeResource(TenantIdentifier tenantIdentifier,
                                                          @Nonnull String key) {
         SingletonResource singletonResource = resources.get(new KeyClass(tenantIdentifier, key));
         if (singletonResource == null) {
@@ -126,18 +118,18 @@ public synchronized SingletonResource removeResource(TenantIdentifier tenantIden
         return singletonResource;
     }
 
-    public synchronized SingletonResource setResource(AppIdentifier appIdentifier,
+    public SingletonResource setResource(AppIdentifier appIdentifier,
                                                       @Nonnull String key,
                                                       SingletonResource resource) {
         return setResource(appIdentifier.getAsPublicTenantIdentifier(), key, resource);
     }
 
-    public synchronized SingletonResource removeResource(AppIdentifier appIdentifier,
+    public SingletonResource removeResource(AppIdentifier appIdentifier,
                                                          @Nonnull String key) {
         return removeResource(appIdentifier.getAsPublicTenantIdentifier(), key);
     }
 
-    public synchronized void clearAllResourcesWithResourceKey(String inputKey) {
+    public void clearAllResourcesWithResourceKey(String inputKey) {
         List<KeyClass> toRemove = new ArrayList<>();
         resources.forEach((key, value) -> {
             if (key.key.equals(inputKey)) {
@@ -149,7 +141,7 @@ public synchronized void clearAllResourcesWithResourceKey(String inputKey) {
         }
     }
 
-    public synchronized Map<KeyClass, SingletonResource> getAllResourcesWithResourceKey(String inputKey) {
+    public Map<KeyClass, SingletonResource> getAllResourcesWithResourceKey(String inputKey) {
         Map<KeyClass, SingletonResource> result = new HashMap<>();
         resources.forEach((key, value) -> {
             if (key.key.equals(inputKey)) {
@@ -160,7 +152,7 @@ public synchronized Map<KeyClass, SingletonResource> getAllResourcesWithResource
     }
 
     @TestOnly
-    public synchronized SingletonResource setResource(@Nonnull String key,
+    public SingletonResource setResource(@Nonnull String key,
                                                       SingletonResource resource) {
         return setResource(appUsedForTesting, key, resource);
     }

ee/jar/ee.jar

@@ -426,6 +426,13 @@ public class CoreConfig {
                     "null)")
     private String otel_collector_connection_uri = null;
 
+    @EnvName("DEADLOCK_LOGGER_ENABLE")
+    @ConfigYamlOnly
+    @JsonProperty
+    @ConfigDescription(
+            "Enables or disables the deadlock logger. (Default: false)")
+    private boolean deadlock_logger_enable = false;
+
     @IgnoreForAnnotationCheck
     private static boolean disableOAuthValidationForTest = false;
 
@@ -681,6 +688,10 @@ public String getOtelCollectorConnectionURI() {
         return otel_collector_connection_uri;
     }
 
+    public boolean isDeadlockLoggerEnabled() {
+        return deadlock_logger_enable;
+    }
+  
     public String getSAMLLegacyACSURL() {
         return saml_legacy_acs_url;
     }